2024-11-13 03:50:11,525 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 03:50:11,538 main DEBUG Took 0.010506 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 03:50:11,538 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 03:50:11,538 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 03:50:11,539 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 03:50:11,540 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,548 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 03:50:11,560 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,562 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,563 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,564 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,565 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,566 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,568 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,568 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,569 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,570 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,571 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,571 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:50:11,571 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,572 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 03:50:11,573 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:50:11,574 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 03:50:11,576 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 03:50:11,577 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 03:50:11,578 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 03:50:11,579 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 03:50:11,588 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 03:50:11,591 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 03:50:11,593 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 03:50:11,593 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 03:50:11,594 main DEBUG createAppenders(={Console}) 2024-11-13 03:50:11,594 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 03:50:11,595 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 03:50:11,595 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 03:50:11,596 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 03:50:11,596 main DEBUG OutputStream closed 2024-11-13 03:50:11,596 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 03:50:11,597 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 03:50:11,597 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 03:50:11,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 03:50:11,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 03:50:11,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 03:50:11,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 03:50:11,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 03:50:11,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 03:50:11,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 03:50:11,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 03:50:11,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 03:50:11,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 03:50:11,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 03:50:11,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 03:50:11,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 03:50:11,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 03:50:11,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 03:50:11,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 03:50:11,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 03:50:11,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 03:50:11,685 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 03:50:11,686 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 03:50:11,686 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 03:50:11,687 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T03:50:11,923 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d 2024-11-13 03:50:11,926 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 03:50:11,926 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T03:50:11,934 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T03:50:11,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T03:50:11,968 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8, deleteOnExit=true 2024-11-13T03:50:11,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T03:50:11,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/test.cache.data in system properties and HBase conf 2024-11-13T03:50:11,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T03:50:11,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.log.dir in system properties and HBase conf 2024-11-13T03:50:11,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T03:50:11,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T03:50:11,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T03:50:12,060 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T03:50:12,155 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T03:50:12,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:50:12,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:50:12,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T03:50:12,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:50:12,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T03:50:12,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T03:50:12,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:50:12,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:50:12,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T03:50:12,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/nfs.dump.dir in system properties and HBase conf 2024-11-13T03:50:12,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/java.io.tmpdir in system properties and HBase conf 2024-11-13T03:50:12,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:50:12,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T03:50:12,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T03:50:13,347 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T03:50:13,426 INFO [Time-limited test {}] log.Log(170): Logging initialized @2610ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T03:50:13,504 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:50:13,571 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:50:13,594 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:50:13,595 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:50:13,597 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:50:13,618 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:50:13,623 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:50:13,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:50:13,835 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/java.io.tmpdir/jetty-localhost-37267-hadoop-hdfs-3_4_1-tests_jar-_-any-7965775331041562494/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:50:13,847 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37267} 2024-11-13T03:50:13,847 INFO [Time-limited test {}] server.Server(415): Started @3032ms 2024-11-13T03:50:14,382 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:50:14,393 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:50:14,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:50:14,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:50:14,396 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:50:14,397 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:50:14,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:50:14,515 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/java.io.tmpdir/jetty-localhost-40633-hadoop-hdfs-3_4_1-tests_jar-_-any-17203193586696304975/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:50:14,516 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40633} 2024-11-13T03:50:14,516 INFO [Time-limited test {}] server.Server(415): Started @3702ms 2024-11-13T03:50:14,572 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:50:15,424 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data2/current/BP-784585066-172.17.0.2-1731469812880/current, will proceed with Du for space computation calculation, 2024-11-13T03:50:15,424 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data1/current/BP-784585066-172.17.0.2-1731469812880/current, will proceed with Du for space computation calculation, 2024-11-13T03:50:15,458 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:50:15,506 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65d1355165f1f554 with lease ID 0xe9838e2913223909: Processing first storage report for DS-d0329276-7ca6-40b8-ac8d-91945a39d5bb from datanode DatanodeRegistration(127.0.0.1:38833, datanodeUuid=98147724-73a0-4f09-96cb-5b744e935ac9, infoPort=36811, infoSecurePort=0, ipcPort=36325, storageInfo=lv=-57;cid=testClusterID;nsid=150504948;c=1731469812880) 2024-11-13T03:50:15,508 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65d1355165f1f554 with lease ID 0xe9838e2913223909: from storage DS-d0329276-7ca6-40b8-ac8d-91945a39d5bb node DatanodeRegistration(127.0.0.1:38833, datanodeUuid=98147724-73a0-4f09-96cb-5b744e935ac9, infoPort=36811, infoSecurePort=0, ipcPort=36325, storageInfo=lv=-57;cid=testClusterID;nsid=150504948;c=1731469812880), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T03:50:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65d1355165f1f554 with lease ID 0xe9838e2913223909: Processing first storage report for DS-c1e7b0bf-8a2f-4678-ad2b-0576a446c4ba from datanode DatanodeRegistration(127.0.0.1:38833, datanodeUuid=98147724-73a0-4f09-96cb-5b744e935ac9, infoPort=36811, infoSecurePort=0, ipcPort=36325, storageInfo=lv=-57;cid=testClusterID;nsid=150504948;c=1731469812880) 2024-11-13T03:50:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65d1355165f1f554 with lease ID 0xe9838e2913223909: from storage DS-c1e7b0bf-8a2f-4678-ad2b-0576a446c4ba node DatanodeRegistration(127.0.0.1:38833, datanodeUuid=98147724-73a0-4f09-96cb-5b744e935ac9, infoPort=36811, infoSecurePort=0, ipcPort=36325, storageInfo=lv=-57;cid=testClusterID;nsid=150504948;c=1731469812880), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:50:15,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d 2024-11-13T03:50:15,632 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/zookeeper_0, clientPort=62776, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T03:50:15,642 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62776 2024-11-13T03:50:15,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:15,658 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:15,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:50:16,255 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e with version=8 2024-11-13T03:50:16,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/hbase-staging 2024-11-13T03:50:16,343 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T03:50:16,552 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ac1a51731247:0 server-side Connection retries=45 2024-11-13T03:50:16,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:16,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:16,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:50:16,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:16,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:50:16,704 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T03:50:16,761 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T03:50:16,774 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T03:50:16,778 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:50:16,807 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 35810 (auto-detected) 2024-11-13T03:50:16,808 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T03:50:16,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33933 2024-11-13T03:50:16,850 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33933 connecting to ZooKeeper ensemble=127.0.0.1:62776 2024-11-13T03:50:16,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339330x0, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:50:16,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33933-0x101339a42000000 connected 2024-11-13T03:50:17,044 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:17,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:17,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:50:17,059 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e, hbase.cluster.distributed=false 2024-11-13T03:50:17,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:50:17,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33933 2024-11-13T03:50:17,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33933 2024-11-13T03:50:17,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33933 2024-11-13T03:50:17,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33933 2024-11-13T03:50:17,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33933 2024-11-13T03:50:17,202 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ac1a51731247:0 server-side Connection retries=45 2024-11-13T03:50:17,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:17,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:17,203 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:50:17,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:50:17,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:50:17,206 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:50:17,209 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:50:17,210 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42123 2024-11-13T03:50:17,212 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42123 connecting to ZooKeeper ensemble=127.0.0.1:62776 2024-11-13T03:50:17,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:17,217 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:17,240 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421230x0, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:50:17,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42123-0x101339a42000001 connected 2024-11-13T03:50:17,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421230x0, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:50:17,245 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:50:17,253 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:50:17,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:50:17,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:50:17,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42123 2024-11-13T03:50:17,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42123 2024-11-13T03:50:17,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42123 2024-11-13T03:50:17,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42123 2024-11-13T03:50:17,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42123 2024-11-13T03:50:17,277 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ac1a51731247:33933 2024-11-13T03:50:17,279 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ac1a51731247,33933,1731469816394 2024-11-13T03:50:17,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:50:17,309 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:50:17,311 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ac1a51731247,33933,1731469816394 2024-11-13T03:50:17,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:17,340 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:50:17,341 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:17,342 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T03:50:17,343 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ac1a51731247,33933,1731469816394 from backup master directory 2024-11-13T03:50:17,350 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:50:17,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ac1a51731247,33933,1731469816394 2024-11-13T03:50:17,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:50:17,351 WARN [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:50:17,351 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ac1a51731247,33933,1731469816394 2024-11-13T03:50:17,354 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T03:50:17,355 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T03:50:17,414 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/hbase.id] with ID: ae6c169b-cfc2-463c-960c-933eaa38c2d3 2024-11-13T03:50:17,415 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/.tmp/hbase.id 2024-11-13T03:50:17,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:50:17,427 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/.tmp/hbase.id]:[hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/hbase.id] 2024-11-13T03:50:17,475 INFO [master/ac1a51731247:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:50:17,482 INFO [master/ac1a51731247:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T03:50:17,503 INFO [master/ac1a51731247:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-13T03:50:17,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:17,526 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:17,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:50:17,558 INFO [master/ac1a51731247:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:50:17,561 INFO [master/ac1a51731247:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T03:50:17,575 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:17,579 INFO [master/ac1a51731247:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:50:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:50:18,028 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store 2024-11-13T03:50:18,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:50:18,451 INFO [master/ac1a51731247:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T03:50:18,455 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:18,456 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:50:18,456 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:50:18,456 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:50:18,458 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:50:18,458 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:50:18,458 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:50:18,460 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731469818456Disabling compacts and flushes for region at 1731469818456Disabling writes for close at 1731469818458 (+2 ms)Writing region close event to WAL at 1731469818458Closed at 1731469818458 2024-11-13T03:50:18,462 WARN [master/ac1a51731247:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/.initializing 2024-11-13T03:50:18,463 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/WALs/ac1a51731247,33933,1731469816394 2024-11-13T03:50:18,472 INFO [master/ac1a51731247:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:50:18,489 INFO [master/ac1a51731247:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ac1a51731247%2C33933%2C1731469816394, suffix=, logDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/WALs/ac1a51731247,33933,1731469816394, archiveDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/oldWALs, maxLogs=10 2024-11-13T03:50:18,513 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/WALs/ac1a51731247,33933,1731469816394/ac1a51731247%2C33933%2C1731469816394.1731469818494, exclude list is [], retry=0 2024-11-13T03:50:18,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38833,DS-d0329276-7ca6-40b8-ac8d-91945a39d5bb,DISK] 2024-11-13T03:50:18,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T03:50:18,569 INFO [master/ac1a51731247:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/WALs/ac1a51731247,33933,1731469816394/ac1a51731247%2C33933%2C1731469816394.1731469818494 2024-11-13T03:50:18,570 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36811:36811)] 2024-11-13T03:50:18,571 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:50:18,571 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:18,575 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,576 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T03:50:18,641 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:18,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:18,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T03:50:18,649 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:18,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:18,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T03:50:18,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:18,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:18,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T03:50:18,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:18,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:18,659 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,663 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,664 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,670 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,670 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,674 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:50:18,678 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:50:18,683 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:50:18,685 INFO [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70601563, jitterRate=0.052045270800590515}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:50:18,693 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731469818589Initializing all the Stores at 1731469818592 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469818592Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469818593 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469818594 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469818594Cleaning up temporary data from old regions at 1731469818670 (+76 ms)Region opened successfully at 1731469818693 (+23 ms) 2024-11-13T03:50:18,695 INFO [master/ac1a51731247:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T03:50:18,732 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e8568f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ac1a51731247/172.17.0.2:0 2024-11-13T03:50:18,763 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T03:50:18,773 INFO [master/ac1a51731247:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T03:50:18,773 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T03:50:18,776 INFO [master/ac1a51731247:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T03:50:18,777 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T03:50:18,782 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T03:50:18,782 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T03:50:18,807 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T03:50:18,819 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T03:50:18,867 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T03:50:18,870 INFO [master/ac1a51731247:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T03:50:18,872 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T03:50:18,883 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T03:50:18,885 INFO [master/ac1a51731247:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T03:50:18,889 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T03:50:18,900 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T03:50:18,902 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T03:50:18,908 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T03:50:18,926 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T03:50:18,933 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T03:50:18,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:50:18,942 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:50:18,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:18,942 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:18,944 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ac1a51731247,33933,1731469816394, sessionid=0x101339a42000000, setting cluster-up flag (Was=false) 2024-11-13T03:50:18,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:18,967 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:18,992 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T03:50:18,994 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ac1a51731247,33933,1731469816394 2024-11-13T03:50:19,017 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:19,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:19,042 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T03:50:19,044 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ac1a51731247,33933,1731469816394 2024-11-13T03:50:19,050 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T03:50:19,068 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(746): ClusterId : ae6c169b-cfc2-463c-960c-933eaa38c2d3 2024-11-13T03:50:19,071 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:50:19,086 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:50:19,086 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:50:19,093 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:50:19,094 DEBUG [RS:0;ac1a51731247:42123 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0d7d57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ac1a51731247/172.17.0.2:0 2024-11-13T03:50:19,108 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ac1a51731247:42123 2024-11-13T03:50:19,111 INFO [RS:0;ac1a51731247:42123 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:50:19,111 INFO [RS:0;ac1a51731247:42123 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:50:19,111 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:50:19,114 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(2659): reportForDuty to master=ac1a51731247,33933,1731469816394 with port=42123, startcode=1731469817167 2024-11-13T03:50:19,125 DEBUG [RS:0;ac1a51731247:42123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:50:19,124 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T03:50:19,135 INFO [master/ac1a51731247:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T03:50:19,143 INFO [master/ac1a51731247:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T03:50:19,149 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ac1a51731247,33933,1731469816394 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T03:50:19,159 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ac1a51731247:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:50:19,159 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ac1a51731247:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:50:19,159 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ac1a51731247:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:50:19,160 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ac1a51731247:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:50:19,160 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ac1a51731247:0, corePoolSize=10, maxPoolSize=10 2024-11-13T03:50:19,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:50:19,160 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,161 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ac1a51731247:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:50:19,161 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,165 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731469849165 2024-11-13T03:50:19,167 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T03:50:19,167 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33933 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:19,168 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T03:50:19,170 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:50:19,171 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T03:50:19,172 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T03:50:19,172 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T03:50:19,173 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T03:50:19,173 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T03:50:19,174 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,177 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:19,177 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T03:50:19,179 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T03:50:19,180 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T03:50:19,181 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T03:50:19,183 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T03:50:19,183 INFO [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T03:50:19,188 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.large.0-1731469819185,5,FailOnTimeoutGroup] 2024-11-13T03:50:19,189 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.small.0-1731469819188,5,FailOnTimeoutGroup] 2024-11-13T03:50:19,189 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,189 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T03:50:19,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:50:19,191 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,192 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,201 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T03:50:19,201 WARN [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T03:50:19,303 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(2659): reportForDuty to master=ac1a51731247,33933,1731469816394 with port=42123, startcode=1731469817167 2024-11-13T03:50:19,306 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,308 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33933 {}] master.ServerManager(517): Registering regionserver=ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,318 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:50:19,318 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44905 2024-11-13T03:50:19,318 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:50:19,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:50:19,334 DEBUG [RS:0;ac1a51731247:42123 {}] zookeeper.ZKUtil(111): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,334 WARN [RS:0;ac1a51731247:42123 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:50:19,334 INFO [RS:0;ac1a51731247:42123 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:50:19,335 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ac1a51731247,42123,1731469817167] 2024-11-13T03:50:19,357 INFO [RS:0;ac1a51731247:42123 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:50:19,372 INFO [RS:0;ac1a51731247:42123 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:50:19,377 INFO [RS:0;ac1a51731247:42123 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:50:19,377 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,378 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:50:19,382 INFO [RS:0;ac1a51731247:42123 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:50:19,384 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,384 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,384 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,384 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,384 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,384 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ac1a51731247:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,385 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ac1a51731247:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:50:19,386 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ac1a51731247:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:50:19,386 DEBUG [RS:0;ac1a51731247:42123 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:50:19,387 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,387 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,387 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,387 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,387 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,388 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,42123,1731469817167-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:50:19,414 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:50:19,417 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,42123,1731469817167-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,417 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,417 INFO [RS:0;ac1a51731247:42123 {}] regionserver.Replication(171): ac1a51731247,42123,1731469817167 started 2024-11-13T03:50:19,435 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:19,436 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1482): Serving as ac1a51731247,42123,1731469817167, RpcServer on ac1a51731247/172.17.0.2:42123, sessionid=0x101339a42000001 2024-11-13T03:50:19,437 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:50:19,437 DEBUG [RS:0;ac1a51731247:42123 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,437 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ac1a51731247,42123,1731469817167' 2024-11-13T03:50:19,437 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:50:19,438 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:50:19,439 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:50:19,439 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:50:19,440 DEBUG [RS:0;ac1a51731247:42123 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ac1a51731247,42123,1731469817167 2024-11-13T03:50:19,440 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ac1a51731247,42123,1731469817167' 2024-11-13T03:50:19,440 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:50:19,440 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:50:19,441 DEBUG [RS:0;ac1a51731247:42123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:50:19,441 INFO [RS:0;ac1a51731247:42123 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:50:19,441 INFO [RS:0;ac1a51731247:42123 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:50:19,547 INFO [RS:0;ac1a51731247:42123 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:50:19,551 INFO [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ac1a51731247%2C42123%2C1731469817167, suffix=, logDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167, archiveDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/oldWALs, maxLogs=32 2024-11-13T03:50:19,568 DEBUG [RS:0;ac1a51731247:42123 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167/ac1a51731247%2C42123%2C1731469817167.1731469819553, exclude list is [], retry=0 2024-11-13T03:50:19,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38833,DS-d0329276-7ca6-40b8-ac8d-91945a39d5bb,DISK] 2024-11-13T03:50:19,576 INFO [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167/ac1a51731247%2C42123%2C1731469817167.1731469819553 2024-11-13T03:50:19,576 DEBUG [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36811:36811)] 2024-11-13T03:50:19,594 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T03:50:19,595 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:50:19,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741833_1009 (size=32) 2024-11-13T03:50:20,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:20,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:50:20,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:50:20,014 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:50:20,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:50:20,018 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:50:20,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:50:20,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:50:20,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:50:20,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:50:20,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740 2024-11-13T03:50:20,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740 2024-11-13T03:50:20,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:50:20,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:50:20,033 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:50:20,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:50:20,040 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:50:20,041 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64216882, jitterRate=-0.04309388995170593}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:50:20,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731469820007Initializing all the Stores at 1731469820010 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820010Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820010Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469820010Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820010Cleaning up temporary data from old regions at 1731469820032 (+22 ms)Region opened successfully at 1731469820043 (+11 ms) 2024-11-13T03:50:20,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:50:20,044 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:50:20,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:50:20,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:50:20,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:50:20,045 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:50:20,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731469820043Disabling compacts and flushes for region at 1731469820043Disabling writes for close at 1731469820044 (+1 ms)Writing region close event to WAL at 1731469820045 (+1 ms)Closed at 1731469820045 2024-11-13T03:50:20,048 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:50:20,049 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T03:50:20,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T03:50:20,062 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:50:20,065 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T03:50:20,221 DEBUG [ac1a51731247:33933 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T03:50:20,234 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:20,240 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ac1a51731247,42123,1731469817167, state=OPENING 2024-11-13T03:50:20,298 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T03:50:20,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:20,308 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:50:20,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:50:20,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:50:20,313 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:50:20,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:20,493 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:50:20,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:50:20,508 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T03:50:20,508 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:50:20,509 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T03:50:20,516 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ac1a51731247%2C42123%2C1731469817167.meta, suffix=.meta, logDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167, archiveDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/oldWALs, maxLogs=32 2024-11-13T03:50:20,533 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167/ac1a51731247%2C42123%2C1731469817167.meta.1731469820518.meta, exclude list is [], retry=0 2024-11-13T03:50:20,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38833,DS-d0329276-7ca6-40b8-ac8d-91945a39d5bb,DISK] 2024-11-13T03:50:20,541 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/WALs/ac1a51731247,42123,1731469817167/ac1a51731247%2C42123%2C1731469817167.meta.1731469820518.meta 2024-11-13T03:50:20,541 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36811:36811)] 2024-11-13T03:50:20,542 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:50:20,543 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T03:50:20,546 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T03:50:20,550 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T03:50:20,555 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T03:50:20,556 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:20,556 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T03:50:20,556 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T03:50:20,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:50:20,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:50:20,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:50:20,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:50:20,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:50:20,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:50:20,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:50:20,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:50:20,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:20,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:50:20,573 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:50:20,574 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740 2024-11-13T03:50:20,577 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740 2024-11-13T03:50:20,580 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:50:20,580 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:50:20,581 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:50:20,584 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:50:20,586 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60843353, jitterRate=-0.09336338937282562}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:50:20,586 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T03:50:20,588 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731469820557Writing region info on filesystem at 1731469820557Initializing all the Stores at 1731469820559 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820559Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820560 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469820560Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731469820560Cleaning up temporary data from old regions at 1731469820580 (+20 ms)Running coprocessor post-open hooks at 1731469820586 (+6 ms)Region opened successfully at 1731469820588 (+2 ms) 2024-11-13T03:50:20,594 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731469820483 2024-11-13T03:50:20,605 DEBUG [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T03:50:20,606 INFO [RS_OPEN_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T03:50:20,607 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:20,610 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ac1a51731247,42123,1731469817167, state=OPEN 2024-11-13T03:50:20,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:50:20,696 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:50:20,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:50:20,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:50:20,698 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:20,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T03:50:20,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ac1a51731247,42123,1731469817167 in 383 msec 2024-11-13T03:50:20,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T03:50:20,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 654 msec 2024-11-13T03:50:20,716 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:50:20,716 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T03:50:20,736 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:20,737 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:20,757 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:20,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:20,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6990 sec 2024-11-13T03:50:20,779 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731469820779, completionTime=-1 2024-11-13T03:50:20,783 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T03:50:20,783 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T03:50:20,810 INFO [master/ac1a51731247:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T03:50:20,810 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731469880810 2024-11-13T03:50:20,811 INFO [master/ac1a51731247:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731469940811 2024-11-13T03:50:20,811 INFO [master/ac1a51731247:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-13T03:50:20,813 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,813 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,813 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,814 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ac1a51731247:33933, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,815 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,815 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,823 DEBUG [master/ac1a51731247:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T03:50:20,840 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.488sec 2024-11-13T03:50:20,842 INFO [master/ac1a51731247:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T03:50:20,843 INFO [master/ac1a51731247:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T03:50:20,844 INFO [master/ac1a51731247:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T03:50:20,844 INFO [master/ac1a51731247:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T03:50:20,845 INFO [master/ac1a51731247:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T03:50:20,845 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:50:20,846 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T03:50:20,854 DEBUG [master/ac1a51731247:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T03:50:20,856 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T03:50:20,856 INFO [master/ac1a51731247:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ac1a51731247,33933,1731469816394-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:50:20,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:20,886 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T03:50:20,886 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T03:50:20,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:20,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:20,899 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672abb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:20,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:20,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:20,913 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:20,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:20,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ac1a51731247,33933,1731469816394 2024-11-13T03:50:20,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=8760 2024-11-13T03:50:20,989 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:50:20,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:50:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@485738bd 2024-11-13T03:50:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:50:21,003 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:50:21,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:50:21,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:50:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:21,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:50:21,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T03:50:21,028 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:21,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:50:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:21,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741835_1011 (size=1061) 2024-11-13T03:50:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:21,446 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:50:21,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741836_1012 (size=53) 2024-11-13T03:50:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:21,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:21,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 40c7504f29b68767a43cbb0fe4bac67e, disabling compactions & flushes 2024-11-13T03:50:21,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:21,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:21,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. after waiting 0 ms 2024-11-13T03:50:21,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:21,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:21,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40c7504f29b68767a43cbb0fe4bac67e: Waiting for close lock at 1731469821859Disabling compacts and flushes for region at 1731469821859Disabling writes for close at 1731469821859Writing region close event to WAL at 1731469821859Closed at 1731469821859 2024-11-13T03:50:21,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:50:21,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469821863"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469821863"}]},"ts":"1731469821863"} 2024-11-13T03:50:21,872 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:50:21,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:50:21,879 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469821875"}]},"ts":"1731469821875"} 2024-11-13T03:50:21,883 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:50:21,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, ASSIGN}] 2024-11-13T03:50:21,888 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, ASSIGN 2024-11-13T03:50:21,890 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:50:22,042 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40c7504f29b68767a43cbb0fe4bac67e, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:22,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, ASSIGN because future has completed 2024-11-13T03:50:22,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:22,212 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:22,212 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:50:22,213 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,213 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:22,213 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,214 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,216 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,219 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:22,220 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40c7504f29b68767a43cbb0fe4bac67e columnFamilyName A 2024-11-13T03:50:22,220 DEBUG [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:22,221 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(327): Store=40c7504f29b68767a43cbb0fe4bac67e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:22,221 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,223 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:22,224 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40c7504f29b68767a43cbb0fe4bac67e columnFamilyName B 2024-11-13T03:50:22,224 DEBUG [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:22,225 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(327): Store=40c7504f29b68767a43cbb0fe4bac67e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:22,225 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,227 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:22,227 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40c7504f29b68767a43cbb0fe4bac67e columnFamilyName C 2024-11-13T03:50:22,227 DEBUG [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:22,228 INFO [StoreOpener-40c7504f29b68767a43cbb0fe4bac67e-1 {}] regionserver.HStore(327): Store=40c7504f29b68767a43cbb0fe4bac67e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:22,228 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:22,229 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,230 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,231 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,233 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,233 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,234 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:50:22,236 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,240 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:50:22,241 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 40c7504f29b68767a43cbb0fe4bac67e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64681683, jitterRate=-0.03616781532764435}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:50:22,241 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:22,243 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 40c7504f29b68767a43cbb0fe4bac67e: Running coprocessor pre-open hook at 1731469822214Writing region info on filesystem at 1731469822214Initializing all the Stores at 1731469822216 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469822216Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469822216Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469822216Cleaning up temporary data from old regions at 1731469822233 (+17 ms)Running coprocessor post-open hooks at 1731469822241 (+8 ms)Region opened successfully at 1731469822243 (+2 ms) 2024-11-13T03:50:22,245 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., pid=6, masterSystemTime=1731469822202 2024-11-13T03:50:22,249 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:22,249 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:22,250 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40c7504f29b68767a43cbb0fe4bac67e, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:22,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:50:22,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T03:50:22,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 in 210 msec 2024-11-13T03:50:22,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T03:50:22,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, ASSIGN in 377 msec 2024-11-13T03:50:22,269 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:50:22,269 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469822269"}]},"ts":"1731469822269"} 2024-11-13T03:50:22,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:50:22,275 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:50:22,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2570 sec 2024-11-13T03:50:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:50:23,188 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:23,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,195 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,197 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a66f173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,199 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,206 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:50:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:50:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c52dcb8 2024-11-13T03:50:23,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:50:23,212 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:50:23,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,228 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,235 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,242 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b41a1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72baa874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,254 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29adf99b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d14565c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605ecfde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,270 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:23,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:23,273 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:23,276 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:23,280 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,281 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,281 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,282 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,284 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,286 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,287 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,289 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,290 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:23,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,294 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:23,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T03:50:23,315 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:23,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:23,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:23,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:50:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:23,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:50:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:23,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:23,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:23,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:23,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:23,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:23,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:23,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469883493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469883504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469883504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469883505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469883512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bc21b9289ff2466f8fcd3c35c2dd1d5e is 50, key is test_row_0/A:col10/1731469823385/Put/seqid=0 2024-11-13T03:50:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741837_1013 (size=12001) 2024-11-13T03:50:23,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bc21b9289ff2466f8fcd3c35c2dd1d5e 2024-11-13T03:50:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469883628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469883630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469883630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469883638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469883639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:23,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/8cdb8bf5ff614019b44c8105a8210dc2 is 50, key is test_row_0/B:col10/1731469823385/Put/seqid=0 2024-11-13T03:50:23,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741838_1014 (size=12001) 2024-11-13T03:50:23,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469883838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469883848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469883849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469883859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469883861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:24,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:24,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/8cdb8bf5ff614019b44c8105a8210dc2 2024-11-13T03:50:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469884158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469884159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469884168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:24,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469884178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469884178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/58f25670d7fb44e4975e93d2a9005c95 is 50, key is test_row_0/C:col10/1731469823385/Put/seqid=0 2024-11-13T03:50:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741839_1015 (size=12001) 2024-11-13T03:50:24,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/58f25670d7fb44e4975e93d2a9005c95 2024-11-13T03:50:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bc21b9289ff2466f8fcd3c35c2dd1d5e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e 2024-11-13T03:50:24,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T03:50:24,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/8cdb8bf5ff614019b44c8105a8210dc2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2 2024-11-13T03:50:24,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T03:50:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/58f25670d7fb44e4975e93d2a9005c95 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95 2024-11-13T03:50:24,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T03:50:24,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 40c7504f29b68767a43cbb0fe4bac67e in 909ms, sequenceid=13, compaction requested=false 2024-11-13T03:50:24,303 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T03:50:24,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:24,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T03:50:24,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:24,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:50:24,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:24,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:24,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:24,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:24,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:24,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/dcad67208a93477e83a107dde72fcb0e is 50, key is test_row_0/A:col10/1731469823505/Put/seqid=0 2024-11-13T03:50:24,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741840_1016 (size=12001) 2024-11-13T03:50:24,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/dcad67208a93477e83a107dde72fcb0e 2024-11-13T03:50:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/68ed3646e1bf4283b39d3ccdd38dccd4 is 50, key is test_row_0/B:col10/1731469823505/Put/seqid=0 2024-11-13T03:50:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:24,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741841_1017 (size=12001) 2024-11-13T03:50:24,472 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/68ed3646e1bf4283b39d3ccdd38dccd4 2024-11-13T03:50:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/007b9eea1c67498895e1b3327e6ea5d8 is 50, key is test_row_0/C:col10/1731469823505/Put/seqid=0 2024-11-13T03:50:24,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741842_1018 (size=12001) 2024-11-13T03:50:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:24,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469884709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469884711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469884713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469884723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469884723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469884828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469884831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469884838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469884838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469884838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:24,929 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/007b9eea1c67498895e1b3327e6ea5d8 2024-11-13T03:50:24,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/dcad67208a93477e83a107dde72fcb0e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e 2024-11-13T03:50:24,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T03:50:24,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/68ed3646e1bf4283b39d3ccdd38dccd4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4 2024-11-13T03:50:24,995 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T03:50:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/007b9eea1c67498895e1b3327e6ea5d8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8 2024-11-13T03:50:25,030 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T03:50:25,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 40c7504f29b68767a43cbb0fe4bac67e in 690ms, sequenceid=33, compaction requested=false 2024-11-13T03:50:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T03:50:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T03:50:25,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T03:50:25,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T03:50:25,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7160 sec 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:25,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7500 sec 2024-11-13T03:50:25,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/85f2650309cb4ff692542648fc78f590 is 50, key is test_row_0/A:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741843_1019 (size=16681) 2024-11-13T03:50:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469885090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469885090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469885093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/85f2650309cb4ff692542648fc78f590 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469885096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469885098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/cf6208e3963a4e5b9d65adb28b2c0bbe is 50, key is test_row_0/B:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741844_1020 (size=12001) 2024-11-13T03:50:25,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/cf6208e3963a4e5b9d65adb28b2c0bbe 2024-11-13T03:50:25,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/8dbc18def97f4178bd5f52800467b34c is 50, key is test_row_0/C:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741845_1021 (size=12001) 2024-11-13T03:50:25,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/8dbc18def97f4178bd5f52800467b34c 2024-11-13T03:50:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469885207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469885208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469885209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469885218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469885218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/85f2650309cb4ff692542648fc78f590 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590 2024-11-13T03:50:25,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590, entries=250, sequenceid=47, filesize=16.3 K 2024-11-13T03:50:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/cf6208e3963a4e5b9d65adb28b2c0bbe as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe 2024-11-13T03:50:25,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T03:50:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/8dbc18def97f4178bd5f52800467b34c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c 2024-11-13T03:50:25,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T03:50:25,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 40c7504f29b68767a43cbb0fe4bac67e in 250ms, sequenceid=47, compaction requested=true 2024-11-13T03:50:25,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:25,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:25,299 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:25,299 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:25,304 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:25,306 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:25,307 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,307 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=35.2 K 2024-11-13T03:50:25,314 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:25,314 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:25,314 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,315 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=39.7 K 2024-11-13T03:50:25,316 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cdb8bf5ff614019b44c8105a8210dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731469823369 2024-11-13T03:50:25,316 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc21b9289ff2466f8fcd3c35c2dd1d5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731469823369 2024-11-13T03:50:25,317 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcad67208a93477e83a107dde72fcb0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731469823494 2024-11-13T03:50:25,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ed3646e1bf4283b39d3ccdd38dccd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731469823494 2024-11-13T03:50:25,317 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85f2650309cb4ff692542648fc78f590, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824699 2024-11-13T03:50:25,318 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6208e3963a4e5b9d65adb28b2c0bbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824708 2024-11-13T03:50:25,352 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T03:50:25,381 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:25,382 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:25,383 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/58fad009f44d447ba2a598a4353eca0d is 50, key is test_row_0/A:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,384 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/471bc9ba50ef4fed8cef67d4a4bb4cd0 is 50, key is test_row_0/B:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741846_1022 (size=12104) 2024-11-13T03:50:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741847_1023 (size=12104) 2024-11-13T03:50:25,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-13T03:50:25,456 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/58fad009f44d447ba2a598a4353eca0d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/58fad009f44d447ba2a598a4353eca0d 2024-11-13T03:50:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:25,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T03:50:25,479 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:25,480 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T03:50:25,485 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 58fad009f44d447ba2a598a4353eca0d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:25,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T03:50:25,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:25,488 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469825295; duration=0sec 2024-11-13T03:50:25,489 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:25,489 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:25,489 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:25,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:25,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:25,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ce782f00f6414a909f2a3dae620eb465 is 50, key is test_row_0/A:col10/1731469825446/Put/seqid=0 2024-11-13T03:50:25,498 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:25,499 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:25,499 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,499 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=35.2 K 2024-11-13T03:50:25,502 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f25670d7fb44e4975e93d2a9005c95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731469823369 2024-11-13T03:50:25,506 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007b9eea1c67498895e1b3327e6ea5d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731469823494 2024-11-13T03:50:25,511 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dbc18def97f4178bd5f52800467b34c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824708 2024-11-13T03:50:25,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741848_1024 (size=12001) 2024-11-13T03:50:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469885498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469885501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ce782f00f6414a909f2a3dae620eb465 2024-11-13T03:50:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469885506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469885514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469885514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb1b858c5ce847358e6093be68a48dea is 50, key is test_row_0/B:col10/1731469825446/Put/seqid=0 2024-11-13T03:50:25,587 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#13 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:25,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7f7d0e0ad7044661991fe1d86cd9b8c9 is 50, key is test_row_0/C:col10/1731469825040/Put/seqid=0 2024-11-13T03:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T03:50:25,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741849_1025 (size=12001) 2024-11-13T03:50:25,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb1b858c5ce847358e6093be68a48dea 2024-11-13T03:50:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741850_1026 (size=12104) 2024-11-13T03:50:25,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469885617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 is 50, key is test_row_0/C:col10/1731469825446/Put/seqid=0 2024-11-13T03:50:25,633 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7f7d0e0ad7044661991fe1d86cd9b8c9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7f7d0e0ad7044661991fe1d86cd9b8c9 2024-11-13T03:50:25,648 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 7f7d0e0ad7044661991fe1d86cd9b8c9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:25,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,649 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469825298; duration=0sec 2024-11-13T03:50:25,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:25,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:25,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469885638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469885647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469885647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469885648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741851_1027 (size=12001) 2024-11-13T03:50:25,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 2024-11-13T03:50:25,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T03:50:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:25,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ce782f00f6414a909f2a3dae620eb465 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465 2024-11-13T03:50:25,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465, entries=150, sequenceid=68, filesize=11.7 K 2024-11-13T03:50:25,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb1b858c5ce847358e6093be68a48dea as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea 2024-11-13T03:50:25,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea, entries=150, sequenceid=68, filesize=11.7 K 2024-11-13T03:50:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 2024-11-13T03:50:25,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2, entries=150, sequenceid=68, filesize=11.7 K 2024-11-13T03:50:25,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 40c7504f29b68767a43cbb0fe4bac67e in 310ms, sequenceid=68, compaction requested=false 2024-11-13T03:50:25,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T03:50:25,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T03:50:25,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:25,820 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:50:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:25,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/5ed64bab0c6d447287a1087daaee503c is 50, key is test_row_0/A:col10/1731469825503/Put/seqid=0 2024-11-13T03:50:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:25,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:25,864 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/471bc9ba50ef4fed8cef67d4a4bb4cd0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/471bc9ba50ef4fed8cef67d4a4bb4cd0 2024-11-13T03:50:25,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741852_1028 (size=12001) 2024-11-13T03:50:25,882 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/5ed64bab0c6d447287a1087daaee503c 2024-11-13T03:50:25,886 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 471bc9ba50ef4fed8cef67d4a4bb4cd0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:25,886 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:25,887 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469825298; duration=0sec 2024-11-13T03:50:25,887 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:25,887 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/d1a7b8aacbce452eb3d385850c858dd6 is 50, key is test_row_0/B:col10/1731469825503/Put/seqid=0 2024-11-13T03:50:25,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469885915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469885917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469885921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469885921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469885930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:25,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741853_1029 (size=12001) 2024-11-13T03:50:25,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/d1a7b8aacbce452eb3d385850c858dd6 2024-11-13T03:50:25,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/86a49e79da00449f818815c41f305d9f is 50, key is test_row_0/C:col10/1731469825503/Put/seqid=0 2024-11-13T03:50:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741854_1030 (size=12001) 2024-11-13T03:50:26,001 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/86a49e79da00449f818815c41f305d9f 2024-11-13T03:50:26,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/5ed64bab0c6d447287a1087daaee503c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c 2024-11-13T03:50:26,040 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T03:50:26,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/d1a7b8aacbce452eb3d385850c858dd6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6 2024-11-13T03:50:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469886037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469886037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469886047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469886048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469886048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,063 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T03:50:26,067 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:50:26,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/86a49e79da00449f818815c41f305d9f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f 2024-11-13T03:50:26,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T03:50:26,089 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 40c7504f29b68767a43cbb0fe4bac67e in 270ms, sequenceid=82, compaction requested=true 2024-11-13T03:50:26,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:26,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T03:50:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T03:50:26,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T03:50:26,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 609 msec 2024-11-13T03:50:26,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 625 msec 2024-11-13T03:50:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T03:50:26,118 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:26,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T03:50:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:26,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:26,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:26,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:26,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-13T03:50:26,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469825886, max timestamp=1731469826259Num uniques 859; 2024-11-13T03:50:26,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469825886, max timestamp=1731469826259Num uniques 859; 2024-11-13T03:50:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:26,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/72d18c685bd54f1dadc035efec8fae51 is 50, key is test_row_0/A:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:26,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469886331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469886331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469886334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469886337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469886346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741855_1031 (size=12001) 2024-11-13T03:50:26,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469886458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469886458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469886459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469886459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469886467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469886678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469886678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469886679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469886684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469886684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:26,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T03:50:26,760 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T03:50:26,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/72d18c685bd54f1dadc035efec8fae51 2024-11-13T03:50:26,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:50:26,776 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T03:50:26,778 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T03:50:26,778 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T03:50:26,782 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T03:50:26,782 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T03:50:26,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4c6d0c57923b4486b33b04c1120582f6 is 50, key is test_row_0/B:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741856_1032 (size=12001) 2024-11-13T03:50:26,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4c6d0c57923b4486b33b04c1120582f6 2024-11-13T03:50:26,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/94cc237205e145b38e51d5a7259d3f7a is 50, key is test_row_0/C:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:26,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741857_1033 (size=12001) 2024-11-13T03:50:26,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/94cc237205e145b38e51d5a7259d3f7a 2024-11-13T03:50:26,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:26,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:26,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/72d18c685bd54f1dadc035efec8fae51 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51 2024-11-13T03:50:26,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T03:50:26,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4c6d0c57923b4486b33b04c1120582f6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6 2024-11-13T03:50:26,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T03:50:26,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/94cc237205e145b38e51d5a7259d3f7a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a 2024-11-13T03:50:26,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T03:50:26,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 40c7504f29b68767a43cbb0fe4bac67e in 727ms, sequenceid=105, compaction requested=true 2024-11-13T03:50:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:26,992 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:26,992 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:26,994 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:26,994 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:26,994 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:26,994 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:26,994 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,995 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:26,995 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/471bc9ba50ef4fed8cef67d4a4bb4cd0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=47.0 K 2024-11-13T03:50:26,995 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/58fad009f44d447ba2a598a4353eca0d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=47.0 K 2024-11-13T03:50:26,996 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58fad009f44d447ba2a598a4353eca0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824708 2024-11-13T03:50:26,996 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 471bc9ba50ef4fed8cef67d4a4bb4cd0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824708 2024-11-13T03:50:26,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce782f00f6414a909f2a3dae620eb465, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731469825440 2024-11-13T03:50:26,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1b858c5ce847358e6093be68a48dea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731469825440 2024-11-13T03:50:26,999 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed64bab0c6d447287a1087daaee503c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731469825497 2024-11-13T03:50:27,000 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a7b8aacbce452eb3d385850c858dd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731469825497 2024-11-13T03:50:27,001 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d18c685bd54f1dadc035efec8fae51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:27,002 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c6d0c57923b4486b33b04c1120582f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:27,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:50:27,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469826327, max timestamp=1731469827000Num uniques 859; 2024-11-13T03:50:27,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:27,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469826327, max timestamp=1731469827000Num uniques 859; 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:27,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,042 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:27,043 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4c92a4c7d2134408adaedd656be5fddd is 50, key is test_row_0/B:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:27,044 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:27,045 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6ca654c0a1c145e2818d98488fdf8664 is 50, key is test_row_0/A:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:27,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8bc6e4b08dbb434d8213f0c6287cd982 is 50, key is test_row_0/A:col10/1731469827000/Put/seqid=0 2024-11-13T03:50:27,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469827056, max timestamp=1731469827087Num uniques 859; 2024-11-13T03:50:27,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469827056, max timestamp=1731469827087Num uniques 859; 2024-11-13T03:50:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:27,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741858_1034 (size=12241) 2024-11-13T03:50:27,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741859_1035 (size=12241) 2024-11-13T03:50:27,114 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6ca654c0a1c145e2818d98488fdf8664 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6ca654c0a1c145e2818d98488fdf8664 2024-11-13T03:50:27,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731469827064, max timestamp=1731469827085Num uniques 859; 2024-11-13T03:50:27,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T03:50:27,129 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 6ca654c0a1c145e2818d98488fdf8664(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:27,129 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,129 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=12, startTime=1731469826992; duration=0sec 2024-11-13T03:50:27,130 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:27,130 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:27,130 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:27,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741860_1036 (size=12001) 2024-11-13T03:50:27,131 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4c92a4c7d2134408adaedd656be5fddd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c92a4c7d2134408adaedd656be5fddd 2024-11-13T03:50:27,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8bc6e4b08dbb434d8213f0c6287cd982 2024-11-13T03:50:27,134 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:27,134 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:27,134 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,135 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7f7d0e0ad7044661991fe1d86cd9b8c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=47.0 K 2024-11-13T03:50:27,137 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7d0e0ad7044661991fe1d86cd9b8c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731469824708 2024-11-13T03:50:27,142 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 861538a8d6ba4fd9bbc68cfa0b9f11c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731469825440 2024-11-13T03:50:27,146 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a49e79da00449f818815c41f305d9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731469825497 2024-11-13T03:50:27,146 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 4c92a4c7d2134408adaedd656be5fddd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:27,146 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,147 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=12, startTime=1731469826992; duration=0sec 2024-11-13T03:50:27,147 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:27,147 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:27,147 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cc237205e145b38e51d5a7259d3f7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:27,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/0c1ae762fb42404db04c512f9fb8378d is 50, key is test_row_0/B:col10/1731469827000/Put/seqid=0 2024-11-13T03:50:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469887155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469887172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469887177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,190 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469887179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469887179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,192 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7bc22652a2d74ac6a491ba357da7e995 is 50, key is test_row_0/C:col10/1731469825917/Put/seqid=0 2024-11-13T03:50:27,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741861_1037 (size=12001) 2024-11-13T03:50:27,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/0c1ae762fb42404db04c512f9fb8378d 2024-11-13T03:50:27,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741862_1038 (size=12241) 2024-11-13T03:50:27,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:27,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:27,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6c0f77b412af4f5f9d54f342df1a0f2e is 50, key is test_row_0/C:col10/1731469827000/Put/seqid=0 2024-11-13T03:50:27,244 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7bc22652a2d74ac6a491ba357da7e995 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7bc22652a2d74ac6a491ba357da7e995 2024-11-13T03:50:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:27,272 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 7bc22652a2d74ac6a491ba357da7e995(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:27,272 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,272 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=12, startTime=1731469826992; duration=0sec 2024-11-13T03:50:27,272 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:27,273 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741863_1039 (size=12001) 2024-11-13T03:50:27,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6c0f77b412af4f5f9d54f342df1a0f2e 2024-11-13T03:50:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469887287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469887287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469887299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469887299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8bc6e4b08dbb434d8213f0c6287cd982 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982 2024-11-13T03:50:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469887304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T03:50:27,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/0c1ae762fb42404db04c512f9fb8378d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d 2024-11-13T03:50:27,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T03:50:27,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6c0f77b412af4f5f9d54f342df1a0f2e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e 2024-11-13T03:50:27,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T03:50:27,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.08 KB/130125 for 40c7504f29b68767a43cbb0fe4bac67e in 353ms, sequenceid=126, compaction requested=false 2024-11-13T03:50:27,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T03:50:27,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,388 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=127.08 KB heapSize=354.77 KB 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/2615cd3c365e4163bd2d136f8b13a416 is 50, key is test_row_0/A:col10/1731469827165/Put/seqid=0 2024-11-13T03:50:27,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741864_1040 (size=12151) 2024-11-13T03:50:27,424 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/2615cd3c365e4163bd2d136f8b13a416 2024-11-13T03:50:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/1759e3a16c0d47689465ccc571a66594 is 50, key is test_row_0/B:col10/1731469827165/Put/seqid=0 2024-11-13T03:50:27,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741865_1041 (size=12151) 2024-11-13T03:50:27,477 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/1759e3a16c0d47689465ccc571a66594 2024-11-13T03:50:27,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/da2c153b737f49558a42eaf9c55bb768 is 50, key is test_row_0/C:col10/1731469827165/Put/seqid=0 2024-11-13T03:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:27,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:27,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469887555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469887555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469887557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741866_1042 (size=12151) 2024-11-13T03:50:27,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/da2c153b737f49558a42eaf9c55bb768 2024-11-13T03:50:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469887564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469887564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/2615cd3c365e4163bd2d136f8b13a416 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416 2024-11-13T03:50:27,599 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T03:50:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/1759e3a16c0d47689465ccc571a66594 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594 2024-11-13T03:50:27,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T03:50:27,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/da2c153b737f49558a42eaf9c55bb768 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768 2024-11-13T03:50:27,635 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T03:50:27,638 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~354.73 KB/363240, currentSize=60.38 KB/61830 for 40c7504f29b68767a43cbb0fe4bac67e in 250ms, sequenceid=163, compaction requested=true 2024-11-13T03:50:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T03:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T03:50:27,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T03:50:27,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5100 sec 2024-11-13T03:50:27,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5370 sec 2024-11-13T03:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:27,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:50:27,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:27,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:27,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:27,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:27,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0f5571541ea34caca1523e24be0f999c is 50, key is test_row_0/A:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469887726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469887730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741867_1043 (size=16931) 2024-11-13T03:50:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469887735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469887730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0f5571541ea34caca1523e24be0f999c 2024-11-13T03:50:27,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/161bcbbbd901421aad5681081c2e3907 is 50, key is test_row_0/B:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741868_1044 (size=12151) 2024-11-13T03:50:27,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/161bcbbbd901421aad5681081c2e3907 2024-11-13T03:50:27,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a07f473a3d9a40c19cdae17e4797d50c is 50, key is test_row_0/C:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:27,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469887847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469887847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469887848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469887848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469887848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:27,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741869_1045 (size=12151) 2024-11-13T03:50:27,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a07f473a3d9a40c19cdae17e4797d50c 2024-11-13T03:50:27,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0f5571541ea34caca1523e24be0f999c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c 2024-11-13T03:50:27,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c, entries=250, sequenceid=177, filesize=16.5 K 2024-11-13T03:50:27,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/161bcbbbd901421aad5681081c2e3907 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907 2024-11-13T03:50:27,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T03:50:27,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a07f473a3d9a40c19cdae17e4797d50c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c 2024-11-13T03:50:27,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T03:50:27,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 40c7504f29b68767a43cbb0fe4bac67e in 239ms, sequenceid=177, compaction requested=true 2024-11-13T03:50:27,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:27,922 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:27,922 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:27,924 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:27,924 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:27,924 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:27,924 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:27,924 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,924 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:27,925 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6ca654c0a1c145e2818d98488fdf8664, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=52.1 K 2024-11-13T03:50:27,925 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c92a4c7d2134408adaedd656be5fddd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=47.4 K 2024-11-13T03:50:27,927 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca654c0a1c145e2818d98488fdf8664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:27,927 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c92a4c7d2134408adaedd656be5fddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:27,928 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1ae762fb42404db04c512f9fb8378d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731469826327 2024-11-13T03:50:27,928 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc6e4b08dbb434d8213f0c6287cd982, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731469826327 2024-11-13T03:50:27,931 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2615cd3c365e4163bd2d136f8b13a416, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731469827148 2024-11-13T03:50:27,931 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1759e3a16c0d47689465ccc571a66594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731469827148 2024-11-13T03:50:27,932 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5571541ea34caca1523e24be0f999c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827547 2024-11-13T03:50:27,933 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 161bcbbbd901421aad5681081c2e3907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827558 2024-11-13T03:50:27,962 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:27,963 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/2387c15b1f24495a8ac0c24e9ec5cb5d is 50, key is test_row_0/A:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:27,974 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:27,975 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4bcd78e6dc994f59b33570526f1bd71e is 50, key is test_row_0/B:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:27,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741870_1046 (size=12527) 2024-11-13T03:50:27,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741871_1047 (size=12527) 2024-11-13T03:50:28,011 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/2387c15b1f24495a8ac0c24e9ec5cb5d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2387c15b1f24495a8ac0c24e9ec5cb5d 2024-11-13T03:50:28,028 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 2387c15b1f24495a8ac0c24e9ec5cb5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:28,028 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,029 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=12, startTime=1731469827922; duration=0sec 2024-11-13T03:50:28,029 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:28,029 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:28,029 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:28,032 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:28,032 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:28,032 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,032 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7bc22652a2d74ac6a491ba357da7e995, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=47.4 K 2024-11-13T03:50:28,033 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc22652a2d74ac6a491ba357da7e995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731469825886 2024-11-13T03:50:28,034 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0f77b412af4f5f9d54f342df1a0f2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731469826327 2024-11-13T03:50:28,035 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2c153b737f49558a42eaf9c55bb768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731469827148 2024-11-13T03:50:28,035 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07f473a3d9a40c19cdae17e4797d50c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827558 2024-11-13T03:50:28,056 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:28,056 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/b001bbfab9cc4225b1021aeb3cb9e16c is 50, key is test_row_0/C:col10/1731469827560/Put/seqid=0 2024-11-13T03:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:28,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:28,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3a55cdacef86418586619ef720e39759 is 50, key is test_row_0/A:col10/1731469828060/Put/seqid=0 2024-11-13T03:50:28,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741872_1048 (size=12527) 2024-11-13T03:50:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,108 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/b001bbfab9cc4225b1021aeb3cb9e16c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/b001bbfab9cc4225b1021aeb3cb9e16c 2024-11-13T03:50:28,118 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into b001bbfab9cc4225b1021aeb3cb9e16c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:28,119 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,119 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=12, startTime=1731469827923; duration=0sec 2024-11-13T03:50:28,119 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:28,119 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:28,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741873_1049 (size=14541) 2024-11-13T03:50:28,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3a55cdacef86418586619ef720e39759 2024-11-13T03:50:28,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/332f89097e85461ea5aa80e8e8b3ac2a is 50, key is test_row_0/B:col10/1731469828060/Put/seqid=0 2024-11-13T03:50:28,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741874_1050 (size=12151) 2024-11-13T03:50:28,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/332f89097e85461ea5aa80e8e8b3ac2a 2024-11-13T03:50:28,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/1dae856e3acc4fd188105fe9a043f5ad is 50, key is test_row_0/C:col10/1731469828060/Put/seqid=0 2024-11-13T03:50:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741875_1051 (size=12151) 2024-11-13T03:50:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/1dae856e3acc4fd188105fe9a043f5ad 2024-11-13T03:50:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3a55cdacef86418586619ef720e39759 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759 2024-11-13T03:50:28,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759, entries=200, sequenceid=197, filesize=14.2 K 2024-11-13T03:50:28,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/332f89097e85461ea5aa80e8e8b3ac2a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a 2024-11-13T03:50:28,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T03:50:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/1dae856e3acc4fd188105fe9a043f5ad as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad 2024-11-13T03:50:28,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T03:50:28,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 40c7504f29b68767a43cbb0fe4bac67e in 217ms, sequenceid=197, compaction requested=false 2024-11-13T03:50:28,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T03:50:28,290 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:28,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T03:50:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T03:50:28,293 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:28,295 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:28,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T03:50:28,408 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4bcd78e6dc994f59b33570526f1bd71e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4bcd78e6dc994f59b33570526f1bd71e 2024-11-13T03:50:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:28,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T03:50:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:28,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,427 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 4bcd78e6dc994f59b33570526f1bd71e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:28,427 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,428 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=12, startTime=1731469827922; duration=0sec 2024-11-13T03:50:28,428 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:28,428 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:28,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f23130b541b7493896acee63ba320fc6 is 50, key is test_row_0/A:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T03:50:28,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741876_1052 (size=16931) 2024-11-13T03:50:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f23130b541b7493896acee63ba320fc6 2024-11-13T03:50:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/399ec091be6f4ec2afe78621a8ee3f9c is 50, key is test_row_0/B:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741877_1053 (size=12151) 2024-11-13T03:50:28,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/399ec091be6f4ec2afe78621a8ee3f9c 2024-11-13T03:50:28,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ac82ac332b25450dba673055c2a3d49c is 50, key is test_row_0/C:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741878_1054 (size=12151) 2024-11-13T03:50:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ac82ac332b25450dba673055c2a3d49c 2024-11-13T03:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T03:50:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T03:50:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f23130b541b7493896acee63ba320fc6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6 2024-11-13T03:50:28,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:28,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6, entries=250, sequenceid=215, filesize=16.5 K 2024-11-13T03:50:28,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/399ec091be6f4ec2afe78621a8ee3f9c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c 2024-11-13T03:50:28,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T03:50:28,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ac82ac332b25450dba673055c2a3d49c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c 2024-11-13T03:50:28,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T03:50:28,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 40c7504f29b68767a43cbb0fe4bac67e in 228ms, sequenceid=215, compaction requested=true 2024-11-13T03:50:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:28,654 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:28,654 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:28,655 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:28,656 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:28,656 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,656 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4bcd78e6dc994f59b33570526f1bd71e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.0 K 2024-11-13T03:50:28,659 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:28,659 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bcd78e6dc994f59b33570526f1bd71e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827558 2024-11-13T03:50:28,660 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:28,660 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,660 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2387c15b1f24495a8ac0c24e9ec5cb5d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=43.0 K 2024-11-13T03:50:28,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 332f89097e85461ea5aa80e8e8b3ac2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731469827705 2024-11-13T03:50:28,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:28,661 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2387c15b1f24495a8ac0c24e9ec5cb5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827558 2024-11-13T03:50:28,661 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 399ec091be6f4ec2afe78621a8ee3f9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828096 2024-11-13T03:50:28,661 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a55cdacef86418586619ef720e39759, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731469827705 2024-11-13T03:50:28,662 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f23130b541b7493896acee63ba320fc6, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828090 2024-11-13T03:50:28,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:28,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:28,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:28,682 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:28,683 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/e7f454b6df744dd5b41c10d715ffcf7d is 50, key is test_row_0/B:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,689 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:28,690 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1357a88be9984268991fba074f6ec045 is 50, key is test_row_0/A:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741880_1056 (size=12629) 2024-11-13T03:50:28,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741879_1055 (size=12629) 2024-11-13T03:50:28,737 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/e7f454b6df744dd5b41c10d715ffcf7d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/e7f454b6df744dd5b41c10d715ffcf7d 2024-11-13T03:50:28,753 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into e7f454b6df744dd5b41c10d715ffcf7d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:28,758 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,758 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469828654; duration=0sec 2024-11-13T03:50:28,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:28,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:28,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:28,763 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:28,764 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:28,764 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,764 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/b001bbfab9cc4225b1021aeb3cb9e16c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.0 K 2024-11-13T03:50:28,765 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b001bbfab9cc4225b1021aeb3cb9e16c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469827558 2024-11-13T03:50:28,766 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dae856e3acc4fd188105fe9a043f5ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731469827705 2024-11-13T03:50:28,766 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ac82ac332b25450dba673055c2a3d49c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828096 2024-11-13T03:50:28,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T03:50:28,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8b68a24c61a94e87906799cc9d606228 is 50, key is test_row_0/A:col10/1731469828471/Put/seqid=0 2024-11-13T03:50:28,786 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:28,787 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/1bd6d35acd394939848dd3e823ca2e6c is 50, key is test_row_0/C:col10/1731469828421/Put/seqid=0 2024-11-13T03:50:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741881_1057 (size=12151) 2024-11-13T03:50:28,795 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8b68a24c61a94e87906799cc9d606228 2024-11-13T03:50:28,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741882_1058 (size=12629) 2024-11-13T03:50:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:28,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:28,828 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/1bd6d35acd394939848dd3e823ca2e6c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1bd6d35acd394939848dd3e823ca2e6c 2024-11-13T03:50:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/adffa3af818848689ea34cbdcf13c1e6 is 50, key is test_row_0/B:col10/1731469828471/Put/seqid=0 2024-11-13T03:50:28,841 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 1bd6d35acd394939848dd3e823ca2e6c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:28,841 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,841 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469828662; duration=0sec 2024-11-13T03:50:28,841 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:28,841 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741883_1059 (size=12151) 2024-11-13T03:50:28,860 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/adffa3af818848689ea34cbdcf13c1e6 2024-11-13T03:50:28,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/bcfdd2ad8e2b4422949ab49327bc03f5 is 50, key is test_row_0/C:col10/1731469828471/Put/seqid=0 2024-11-13T03:50:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T03:50:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741884_1060 (size=12151) 2024-11-13T03:50:28,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/bcfdd2ad8e2b4422949ab49327bc03f5 2024-11-13T03:50:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/8b68a24c61a94e87906799cc9d606228 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228 2024-11-13T03:50:28,950 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T03:50:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/adffa3af818848689ea34cbdcf13c1e6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6 2024-11-13T03:50:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469888957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469888968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469888968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469888968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469888968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:28,981 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T03:50:28,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/bcfdd2ad8e2b4422949ab49327bc03f5 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5 2024-11-13T03:50:28,994 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T03:50:28,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 40c7504f29b68767a43cbb0fe4bac67e in 226ms, sequenceid=233, compaction requested=false 2024-11-13T03:50:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T03:50:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T03:50:29,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T03:50:29,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-13T03:50:29,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 716 msec 2024-11-13T03:50:29,124 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1357a88be9984268991fba074f6ec045 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1357a88be9984268991fba074f6ec045 2024-11-13T03:50:29,133 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 1357a88be9984268991fba074f6ec045(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:29,133 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:29,133 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469828654; duration=0sec 2024-11-13T03:50:29,134 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:29,134 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:29,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/7c67071b8fa9430abf695643ac1cfa59 is 50, key is test_row_0/A:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741885_1061 (size=12151) 2024-11-13T03:50:29,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/7c67071b8fa9430abf695643ac1cfa59 2024-11-13T03:50:29,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469889199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469889201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469889204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469889201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469889204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/08130fd61e3d4fbfb2ddcba9185cb4f2 is 50, key is test_row_0/B:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741886_1062 (size=12151) 2024-11-13T03:50:29,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/08130fd61e3d4fbfb2ddcba9185cb4f2 2024-11-13T03:50:29,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a14b4822c0d5472d934d121adc0369c3 is 50, key is test_row_0/C:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741887_1063 (size=12151) 2024-11-13T03:50:29,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a14b4822c0d5472d934d121adc0369c3 2024-11-13T03:50:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/7c67071b8fa9430abf695643ac1cfa59 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59 2024-11-13T03:50:29,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T03:50:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469889318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/08130fd61e3d4fbfb2ddcba9185cb4f2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2 2024-11-13T03:50:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469889319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469889319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469889319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469889319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T03:50:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a14b4822c0d5472d934d121adc0369c3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3 2024-11-13T03:50:29,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T03:50:29,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 40c7504f29b68767a43cbb0fe4bac67e in 192ms, sequenceid=251, compaction requested=true 2024-11-13T03:50:29,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:29,364 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T03:50:29,366 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:29,366 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:29,366 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,366 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1357a88be9984268991fba074f6ec045, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.1 K 2024-11-13T03:50:29,368 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:29,368 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1357a88be9984268991fba074f6ec045, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828096 2024-11-13T03:50:29,369 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b68a24c61a94e87906799cc9d606228, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731469828466 2024-11-13T03:50:29,369 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c67071b8fa9430abf695643ac1cfa59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:29,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:29,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:29,373 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,373 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/e7f454b6df744dd5b41c10d715ffcf7d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.1 K 2024-11-13T03:50:29,378 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f454b6df744dd5b41c10d715ffcf7d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828096 2024-11-13T03:50:29,380 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting adffa3af818848689ea34cbdcf13c1e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731469828466 2024-11-13T03:50:29,382 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 08130fd61e3d4fbfb2ddcba9185cb4f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:29,392 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:29,393 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/9af7db1a2ad04759823f01d5069e344a is 50, key is test_row_0/A:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,403 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:29,404 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/bdfe18f68a384d3fa32072f5f000f492 is 50, key is test_row_0/B:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741888_1064 (size=12731) 2024-11-13T03:50:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T03:50:29,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:29,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T03:50:29,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741889_1065 (size=12731) 2024-11-13T03:50:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:29,441 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:29,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:29,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:29,463 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/bdfe18f68a384d3fa32072f5f000f492 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/bdfe18f68a384d3fa32072f5f000f492 2024-11-13T03:50:29,478 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into bdfe18f68a384d3fa32072f5f000f492(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:29,478 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:29,478 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469829364; duration=0sec 2024-11-13T03:50:29,478 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:29,478 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:29,478 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:29,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:29,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:29,483 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,483 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1bd6d35acd394939848dd3e823ca2e6c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.1 K 2024-11-13T03:50:29,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd6d35acd394939848dd3e823ca2e6c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731469828096 2024-11-13T03:50:29,491 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfdd2ad8e2b4422949ab49327bc03f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731469828466 2024-11-13T03:50:29,491 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a14b4822c0d5472d934d121adc0369c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:29,506 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:29,507 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/2f437c5263fb495aa0270749c98dea87 is 50, key is test_row_0/C:col10/1731469828850/Put/seqid=0 2024-11-13T03:50:29,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T03:50:29,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:29,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741890_1066 (size=12731) 2024-11-13T03:50:29,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3f730ab0724249319d6d1b7c09e1dac0 is 50, key is test_row_0/A:col10/1731469829194/Put/seqid=0 2024-11-13T03:50:29,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469889571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469889573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469889575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469889576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741891_1067 (size=12301) 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469889579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:29,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469889688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469889688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:29,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,827 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/9af7db1a2ad04759823f01d5069e344a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/9af7db1a2ad04759823f01d5069e344a 2024-11-13T03:50:29,847 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 9af7db1a2ad04759823f01d5069e344a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:29,847 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:29,847 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469829363; duration=0sec 2024-11-13T03:50:29,848 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:29,848 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469889897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469889898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469889898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469889898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469889903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:29,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:29,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:29,966 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/2f437c5263fb495aa0270749c98dea87 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/2f437c5263fb495aa0270749c98dea87 2024-11-13T03:50:29,978 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 2f437c5263fb495aa0270749c98dea87(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:29,978 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:29,979 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469829364; duration=0sec 2024-11-13T03:50:29,979 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:29,979 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:29,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3f730ab0724249319d6d1b7c09e1dac0 2024-11-13T03:50:30,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4ea6d91b5f38473782ff121a974bc477 is 50, key is test_row_0/B:col10/1731469829194/Put/seqid=0 2024-11-13T03:50:30,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741892_1068 (size=12301) 2024-11-13T03:50:30,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4ea6d91b5f38473782ff121a974bc477 2024-11-13T03:50:30,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/285458940d764260b80288fda83f5e39 is 50, key is test_row_0/C:col10/1731469829194/Put/seqid=0 2024-11-13T03:50:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741893_1069 (size=12301) 2024-11-13T03:50:30,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/285458940d764260b80288fda83f5e39 2024-11-13T03:50:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:30,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3f730ab0724249319d6d1b7c09e1dac0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0 2024-11-13T03:50:30,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:50:30,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4ea6d91b5f38473782ff121a974bc477 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477 2024-11-13T03:50:30,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:50:30,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/285458940d764260b80288fda83f5e39 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39 2024-11-13T03:50:30,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:50:30,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 40c7504f29b68767a43cbb0fe4bac67e in 570ms, sequenceid=269, compaction requested=false 2024-11-13T03:50:30,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:30,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:30,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/4d3f1a79c30e41cf92ea7576db26e0c3 is 50, key is test_row_0/A:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469890227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469890235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469890236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741894_1070 (size=14741) 2024-11-13T03:50:30,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/4d3f1a79c30e41cf92ea7576db26e0c3 2024-11-13T03:50:30,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ff08f3fb8b024f15afda8c136009fbcf is 50, key is test_row_0/B:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741895_1071 (size=12301) 2024-11-13T03:50:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469890349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469890350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469890351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469890349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469890358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469890556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469890557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469890567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469890567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469890568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:30,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:30,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ff08f3fb8b024f15afda8c136009fbcf 2024-11-13T03:50:30,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e21cb3ba0c2f4ea88608812556318c15 is 50, key is test_row_0/C:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741896_1072 (size=12301) 2024-11-13T03:50:30,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e21cb3ba0c2f4ea88608812556318c15 2024-11-13T03:50:30,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/4d3f1a79c30e41cf92ea7576db26e0c3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3 2024-11-13T03:50:30,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3, entries=200, sequenceid=288, filesize=14.4 K 2024-11-13T03:50:30,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ff08f3fb8b024f15afda8c136009fbcf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf 2024-11-13T03:50:30,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T03:50:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e21cb3ba0c2f4ea88608812556318c15 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15 2024-11-13T03:50:30,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T03:50:30,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 40c7504f29b68767a43cbb0fe4bac67e in 621ms, sequenceid=288, compaction requested=true 2024-11-13T03:50:30,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:30,833 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:30,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:30,834 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:30,835 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:30,835 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:30,835 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,835 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/9af7db1a2ad04759823f01d5069e344a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=38.8 K 2024-11-13T03:50:30,836 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af7db1a2ad04759823f01d5069e344a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:30,837 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f730ab0724249319d6d1b7c09e1dac0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469829194 2024-11-13T03:50:30,837 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:30,837 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:30,837 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,837 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/bdfe18f68a384d3fa32072f5f000f492, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.5 K 2024-11-13T03:50:30,837 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3f1a79c30e41cf92ea7576db26e0c3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:30,838 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfe18f68a384d3fa32072f5f000f492, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:30,839 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea6d91b5f38473782ff121a974bc477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469829194 2024-11-13T03:50:30,839 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ff08f3fb8b024f15afda8c136009fbcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:30,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:30,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T03:50:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,848 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:30,861 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:30,861 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f5c878bbfbe846af912049e457aa9e80 is 50, key is test_row_0/A:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,862 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:30,862 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/0658edfce3144921be24c6ce5fc1b032 is 50, key is test_row_0/B:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3813797018f94e1991303a772181f579 is 50, key is test_row_0/A:col10/1731469830235/Put/seqid=0 2024-11-13T03:50:30,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:30,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741897_1073 (size=12983) 2024-11-13T03:50:30,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741898_1074 (size=12301) 2024-11-13T03:50:30,905 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3813797018f94e1991303a772181f579 2024-11-13T03:50:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469890895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469890899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469890901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469890903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741899_1075 (size=12983) 2024-11-13T03:50:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469890906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:30,926 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/0658edfce3144921be24c6ce5fc1b032 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0658edfce3144921be24c6ce5fc1b032 2024-11-13T03:50:30,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/24b95047103e41e9a1cfcca48712ad89 is 50, key is test_row_0/B:col10/1731469830235/Put/seqid=0 2024-11-13T03:50:30,934 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 0658edfce3144921be24c6ce5fc1b032(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:30,934 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:30,934 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469830834; duration=0sec 2024-11-13T03:50:30,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:30,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:30,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:30,936 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:30,936 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:30,936 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:30,937 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/2f437c5263fb495aa0270749c98dea87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.5 K 2024-11-13T03:50:30,937 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f437c5263fb495aa0270749c98dea87, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469828838 2024-11-13T03:50:30,940 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 285458940d764260b80288fda83f5e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469829194 2024-11-13T03:50:30,941 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e21cb3ba0c2f4ea88608812556318c15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:30,967 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#64 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741900_1076 (size=12301) 2024-11-13T03:50:30,969 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e87bf9a3aed947299e633d5078dc39b8 is 50, key is test_row_0/C:col10/1731469829573/Put/seqid=0 2024-11-13T03:50:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741901_1077 (size=12983) 2024-11-13T03:50:30,995 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e87bf9a3aed947299e633d5078dc39b8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e87bf9a3aed947299e633d5078dc39b8 2024-11-13T03:50:31,005 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into e87bf9a3aed947299e633d5078dc39b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:31,005 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:31,005 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469830834; duration=0sec 2024-11-13T03:50:31,005 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:31,005 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469891017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469891017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469891018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469891018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469891027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469891226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469891227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469891227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469891227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469891236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,295 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f5c878bbfbe846af912049e457aa9e80 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f5c878bbfbe846af912049e457aa9e80 2024-11-13T03:50:31,302 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into f5c878bbfbe846af912049e457aa9e80(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:31,302 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:31,302 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469830833; duration=0sec 2024-11-13T03:50:31,302 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:31,302 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:31,369 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/24b95047103e41e9a1cfcca48712ad89 2024-11-13T03:50:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/be4ae693ca844b4fa5a29628f7ef0f33 is 50, key is test_row_0/C:col10/1731469830235/Put/seqid=0 2024-11-13T03:50:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741902_1078 (size=12301) 2024-11-13T03:50:31,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469891537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469891538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469891538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469891547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469891547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:31,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/be4ae693ca844b4fa5a29628f7ef0f33 2024-11-13T03:50:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3813797018f94e1991303a772181f579 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579 2024-11-13T03:50:31,803 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T03:50:31,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/24b95047103e41e9a1cfcca48712ad89 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89 2024-11-13T03:50:31,813 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T03:50:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/be4ae693ca844b4fa5a29628f7ef0f33 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33 2024-11-13T03:50:31,821 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T03:50:31,822 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 40c7504f29b68767a43cbb0fe4bac67e in 974ms, sequenceid=303, compaction requested=false 2024-11-13T03:50:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:31,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T03:50:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T03:50:31,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T03:50:31,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3810 sec 2024-11-13T03:50:31,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4000 sec 2024-11-13T03:50:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:32,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T03:50:32,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:32,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:32,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/83ecf88d29244882a36fb1ade3a5446c is 50, key is test_row_0/A:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741903_1079 (size=14741) 2024-11-13T03:50:32,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/83ecf88d29244882a36fb1ade3a5446c 2024-11-13T03:50:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469892068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469892068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469892070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469892070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3115ef70e29a4f7fb617ee6b99a23467 is 50, key is test_row_0/B:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469892071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741904_1080 (size=12301) 2024-11-13T03:50:32,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469892187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469892187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469892187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469892188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469892188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469892398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469892398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469892398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469892398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469892398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3115ef70e29a4f7fb617ee6b99a23467 2024-11-13T03:50:32,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a95f1d327d9647f686207221f81ac3c4 is 50, key is test_row_0/C:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741905_1081 (size=12301) 2024-11-13T03:50:32,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a95f1d327d9647f686207221f81ac3c4 2024-11-13T03:50:32,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/83ecf88d29244882a36fb1ade3a5446c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c 2024-11-13T03:50:32,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c, entries=200, sequenceid=326, filesize=14.4 K 2024-11-13T03:50:32,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3115ef70e29a4f7fb617ee6b99a23467 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467 2024-11-13T03:50:32,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T03:50:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/a95f1d327d9647f686207221f81ac3c4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4 2024-11-13T03:50:32,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T03:50:32,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 40c7504f29b68767a43cbb0fe4bac67e in 504ms, sequenceid=326, compaction requested=true 2024-11-13T03:50:32,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:32,557 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:32,557 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:32,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:32,559 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:32,559 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:32,559 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:32,560 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0658edfce3144921be24c6ce5fc1b032, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.7 K 2024-11-13T03:50:32,560 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:32,560 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:32,560 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:32,561 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f5c878bbfbe846af912049e457aa9e80, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=39.1 K 2024-11-13T03:50:32,562 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c878bbfbe846af912049e457aa9e80, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:32,562 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0658edfce3144921be24c6ce5fc1b032, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:32,562 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3813797018f94e1991303a772181f579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731469830218 2024-11-13T03:50:32,563 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b95047103e41e9a1cfcca48712ad89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731469830218 2024-11-13T03:50:32,563 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ecf88d29244882a36fb1ade3a5446c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469830904 2024-11-13T03:50:32,563 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3115ef70e29a4f7fb617ee6b99a23467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469832050 2024-11-13T03:50:32,589 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:32,590 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bf13417d19fd4bcea213fd613e0b55ac is 50, key is test_row_0/A:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,596 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:32,597 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/a653f2f492f44017a8729c27cbb79ef9 is 50, key is test_row_0/B:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741906_1082 (size=13085) 2024-11-13T03:50:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741907_1083 (size=13085) 2024-11-13T03:50:32,637 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/a653f2f492f44017a8729c27cbb79ef9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a653f2f492f44017a8729c27cbb79ef9 2024-11-13T03:50:32,646 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into a653f2f492f44017a8729c27cbb79ef9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:32,646 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:32,646 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469832557; duration=0sec 2024-11-13T03:50:32,646 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:32,646 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:32,646 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:32,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:32,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:32,649 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:32,650 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e87bf9a3aed947299e633d5078dc39b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.7 K 2024-11-13T03:50:32,650 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e87bf9a3aed947299e633d5078dc39b8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731469829573 2024-11-13T03:50:32,651 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting be4ae693ca844b4fa5a29628f7ef0f33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731469830218 2024-11-13T03:50:32,651 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a95f1d327d9647f686207221f81ac3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469832050 2024-11-13T03:50:32,665 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#71 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:32,665 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6aee5e56e6374ce1bd9606d6464f2bec is 50, key is test_row_0/C:col10/1731469832052/Put/seqid=0 2024-11-13T03:50:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741908_1084 (size=13085) 2024-11-13T03:50:32,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6aee5e56e6374ce1bd9606d6464f2bec as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6aee5e56e6374ce1bd9606d6464f2bec 2024-11-13T03:50:32,702 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 6aee5e56e6374ce1bd9606d6464f2bec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:32,703 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:32,703 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469832557; duration=0sec 2024-11-13T03:50:32,703 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:32,703 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:32,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:50:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:32,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/18886022534c4a909c6960bb2fe76074 is 50, key is test_row_0/A:col10/1731469832068/Put/seqid=0 2024-11-13T03:50:32,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741909_1085 (size=14741) 2024-11-13T03:50:32,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/18886022534c4a909c6960bb2fe76074 2024-11-13T03:50:32,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469892735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469892737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469892738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469892739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469892735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b37817606d014e8fa9f612acf4739492 is 50, key is test_row_0/B:col10/1731469832068/Put/seqid=0 2024-11-13T03:50:32,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741910_1086 (size=12301) 2024-11-13T03:50:32,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b37817606d014e8fa9f612acf4739492 2024-11-13T03:50:32,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/0dece04818c44d869f66199e53d303ff is 50, key is test_row_0/C:col10/1731469832068/Put/seqid=0 2024-11-13T03:50:32,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741911_1087 (size=12301) 2024-11-13T03:50:32,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/0dece04818c44d869f66199e53d303ff 2024-11-13T03:50:32,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/18886022534c4a909c6960bb2fe76074 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074 2024-11-13T03:50:32,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074, entries=200, sequenceid=343, filesize=14.4 K 2024-11-13T03:50:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b37817606d014e8fa9f612acf4739492 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492 2024-11-13T03:50:32,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T03:50:32,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/0dece04818c44d869f66199e53d303ff as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff 2024-11-13T03:50:32,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T03:50:32,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 40c7504f29b68767a43cbb0fe4bac67e in 139ms, sequenceid=343, compaction requested=false 2024-11-13T03:50:32,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:32,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:50:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469832729, max timestamp=1731469832854Num uniques 859; 2024-11-13T03:50:32,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469832729, max timestamp=1731469832854Num uniques 859; 2024-11-13T03:50:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:32,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:32,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:32,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:32,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0d53e180d666406180c556b712814b9c is 50, key is test_row_0/A:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:32,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741912_1088 (size=14741) 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469832888, max timestamp=1731469832900Num uniques 859; 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469832888, max timestamp=1731469832900Num uniques 859; 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469832888, max timestamp=1731469832900Num uniques 859; 2024-11-13T03:50:32,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469892959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469892958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469892973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469892976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469892976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,031 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bf13417d19fd4bcea213fd613e0b55ac as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf13417d19fd4bcea213fd613e0b55ac 2024-11-13T03:50:33,051 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into bf13417d19fd4bcea213fd613e0b55ac(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:33,051 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:33,051 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469832557; duration=0sec 2024-11-13T03:50:33,052 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:33,052 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469893090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469893090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469893091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469893091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469893092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0d53e180d666406180c556b712814b9c 2024-11-13T03:50:33,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae is 50, key is test_row_0/B:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469893298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469893299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469893299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469893299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469893309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741913_1089 (size=12301) 2024-11-13T03:50:33,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae 2024-11-13T03:50:33,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/98938c26571d430ebe73a045b9042624 is 50, key is test_row_0/C:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:33,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741914_1090 (size=12301) 2024-11-13T03:50:33,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/98938c26571d430ebe73a045b9042624 2024-11-13T03:50:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0d53e180d666406180c556b712814b9c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c 2024-11-13T03:50:33,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c, entries=200, sequenceid=364, filesize=14.4 K 2024-11-13T03:50:33,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae 2024-11-13T03:50:33,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T03:50:33,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/98938c26571d430ebe73a045b9042624 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624 2024-11-13T03:50:33,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T03:50:33,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=134.18 KB/137400 for 40c7504f29b68767a43cbb0fe4bac67e in 529ms, sequenceid=364, compaction requested=true 2024-11-13T03:50:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:33,386 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:33,386 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:33,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:33,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:33,388 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:33,388 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:33,388 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,389 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a653f2f492f44017a8729c27cbb79ef9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.8 K 2024-11-13T03:50:33,390 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:33,390 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a653f2f492f44017a8729c27cbb79ef9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469832050 2024-11-13T03:50:33,390 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:33,390 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,390 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf13417d19fd4bcea213fd613e0b55ac, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=41.6 K 2024-11-13T03:50:33,390 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b37817606d014e8fa9f612acf4739492, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731469832055 2024-11-13T03:50:33,394 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1d4b2e9b7c43bdb5e62f0efe9c42ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832850 2024-11-13T03:50:33,394 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf13417d19fd4bcea213fd613e0b55ac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469832050 2024-11-13T03:50:33,395 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18886022534c4a909c6960bb2fe76074, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731469832055 2024-11-13T03:50:33,396 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d53e180d666406180c556b712814b9c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832729 2024-11-13T03:50:33,409 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:33,410 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/fae4ae6242964e9fb4870276f8e4b430 is 50, key is test_row_0/A:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:33,417 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:33,417 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2df1336737ba4af2bd49441f9efbba82 is 50, key is test_row_0/B:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:33,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741915_1091 (size=13187) 2024-11-13T03:50:33,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741916_1092 (size=13187) 2024-11-13T03:50:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T03:50:33,607 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:33,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T03:50:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:33,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:33,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:33,611 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:33,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:33,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/777f54f1b3b64087bfa9d0f56f37194c is 50, key is test_row_0/A:col10/1731469832970/Put/seqid=0 2024-11-13T03:50:33,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469893627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741917_1093 (size=14741) 2024-11-13T03:50:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469893629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469893737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469893737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469893738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469893738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469893738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T03:50:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,830 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/fae4ae6242964e9fb4870276f8e4b430 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/fae4ae6242964e9fb4870276f8e4b430 2024-11-13T03:50:33,839 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2df1336737ba4af2bd49441f9efbba82 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2df1336737ba4af2bd49441f9efbba82 2024-11-13T03:50:33,843 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into fae4ae6242964e9fb4870276f8e4b430(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:33,843 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:33,844 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469833386; duration=0sec 2024-11-13T03:50:33,844 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:33,844 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:33,844 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:33,845 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:33,845 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:33,845 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,846 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6aee5e56e6374ce1bd9606d6464f2bec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=36.8 K 2024-11-13T03:50:33,846 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aee5e56e6374ce1bd9606d6464f2bec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731469832050 2024-11-13T03:50:33,846 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dece04818c44d869f66199e53d303ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731469832055 2024-11-13T03:50:33,847 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98938c26571d430ebe73a045b9042624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832850 2024-11-13T03:50:33,849 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 2df1336737ba4af2bd49441f9efbba82(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:33,850 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:33,850 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469833386; duration=0sec 2024-11-13T03:50:33,851 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:33,851 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:33,862 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:33,862 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/4beefbe3185a470baec0323bde895890 is 50, key is test_row_0/C:col10/1731469832854/Put/seqid=0 2024-11-13T03:50:33,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741918_1094 (size=13187) 2024-11-13T03:50:33,909 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/4beefbe3185a470baec0323bde895890 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/4beefbe3185a470baec0323bde895890 2024-11-13T03:50:33,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T03:50:33,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:33,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:33,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:33,928 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 4beefbe3185a470baec0323bde895890(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:33,928 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:33,928 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469833386; duration=0sec 2024-11-13T03:50:33,928 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:33,928 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469893947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469893948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469893948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469893948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469893948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/777f54f1b3b64087bfa9d0f56f37194c 2024-11-13T03:50:34,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/a3d1ca08bc564a9b9e9de651119a1628 is 50, key is test_row_0/B:col10/1731469832970/Put/seqid=0 2024-11-13T03:50:34,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741919_1095 (size=12301) 2024-11-13T03:50:34,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/a3d1ca08bc564a9b9e9de651119a1628 2024-11-13T03:50:34,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f7172ee52ac74eb882bf72c22e524f51 is 50, key is test_row_0/C:col10/1731469832970/Put/seqid=0 2024-11-13T03:50:34,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741920_1096 (size=12301) 2024-11-13T03:50:34,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f7172ee52ac74eb882bf72c22e524f51 2024-11-13T03:50:34,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T03:50:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/777f54f1b3b64087bfa9d0f56f37194c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c 2024-11-13T03:50:34,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:34,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:34,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:34,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:34,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c, entries=200, sequenceid=400, filesize=14.4 K 2024-11-13T03:50:34,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/a3d1ca08bc564a9b9e9de651119a1628 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628 2024-11-13T03:50:34,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628, entries=150, sequenceid=400, filesize=12.0 K 2024-11-13T03:50:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-13T03:50:34,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f7172ee52ac74eb882bf72c22e524f51 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51 2024-11-13T03:50:34,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51, entries=150, sequenceid=400, filesize=12.0 K 2024-11-13T03:50:34,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 40c7504f29b68767a43cbb0fe4bac67e in 518ms, sequenceid=400, compaction requested=false 2024-11-13T03:50:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:34,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T03:50:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:34,238 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:34,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/eeb2355ed42c482e9874434994bc2c1a is 50, key is test_row_0/A:col10/1731469833622/Put/seqid=0 2024-11-13T03:50:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:34,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:34,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741921_1097 (size=12301) 2024-11-13T03:50:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469894295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469894295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469894297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469834279, max timestamp=1731469834307Num uniques 859; 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469834279, max timestamp=1731469834307Num uniques 859; 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469834279, max timestamp=1731469834307Num uniques 859; 2024-11-13T03:50:34,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469894389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469894391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469894418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469894418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469894418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469894507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469894507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469894626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469894627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469894627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,683 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/eeb2355ed42c482e9874434994bc2c1a 2024-11-13T03:50:34,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/021c736bf1c34b5baf5869ca6cefc43c is 50, key is test_row_0/B:col10/1731469833622/Put/seqid=0 2024-11-13T03:50:34,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741922_1098 (size=12301) 2024-11-13T03:50:34,699 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/021c736bf1c34b5baf5869ca6cefc43c 2024-11-13T03:50:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ecddfbaeedcd4ff99114d14bc671ac67 is 50, key is test_row_0/C:col10/1731469833622/Put/seqid=0 2024-11-13T03:50:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741923_1099 (size=12301) 2024-11-13T03:50:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469894727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469894727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:34,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469894937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469894937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469894937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469895037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469895037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ecddfbaeedcd4ff99114d14bc671ac67 2024-11-13T03:50:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/eeb2355ed42c482e9874434994bc2c1a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a 2024-11-13T03:50:35,124 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T03:50:35,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/021c736bf1c34b5baf5869ca6cefc43c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c 2024-11-13T03:50:35,131 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T03:50:35,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ecddfbaeedcd4ff99114d14bc671ac67 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67 2024-11-13T03:50:35,137 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T03:50:35,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 40c7504f29b68767a43cbb0fe4bac67e in 900ms, sequenceid=414, compaction requested=true 2024-11-13T03:50:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T03:50:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T03:50:35,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T03:50:35,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-11-13T03:50:35,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5350 sec 2024-11-13T03:50:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:35,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-13T03:50:35,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:35,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:35,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:35,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:35,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:35,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:35,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bf09f694447643c28985a048b4334523 is 50, key is test_row_0/A:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741924_1100 (size=14741) 2024-11-13T03:50:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469895463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bf09f694447643c28985a048b4334523 2024-11-13T03:50:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469895464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469895466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/67ede1c4d1954d5ba42d3cdfb155abc1 is 50, key is test_row_0/B:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741925_1101 (size=12301) 2024-11-13T03:50:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469895547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469895547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469895577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469895577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469895577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T03:50:35,767 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:35,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-13T03:50:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:35,771 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:35,772 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:35,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:35,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469895788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469895788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469895788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:35,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:35,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/67ede1c4d1954d5ba42d3cdfb155abc1 2024-11-13T03:50:35,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7b02d8e6863740c982b4ff0a61ff8baf is 50, key is test_row_0/C:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:35,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T03:50:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:35,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:35,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741926_1102 (size=12301) 2024-11-13T03:50:35,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7b02d8e6863740c982b4ff0a61ff8baf 2024-11-13T03:50:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:35,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/bf09f694447643c28985a048b4334523 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523 2024-11-13T03:50:35,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523, entries=200, sequenceid=448, filesize=14.4 K 2024-11-13T03:50:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/67ede1c4d1954d5ba42d3cdfb155abc1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1 2024-11-13T03:50:35,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1, entries=150, sequenceid=448, filesize=12.0 K 2024-11-13T03:50:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/7b02d8e6863740c982b4ff0a61ff8baf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf 2024-11-13T03:50:35,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf, entries=150, sequenceid=448, filesize=12.0 K 2024-11-13T03:50:35,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 40c7504f29b68767a43cbb0fe4bac67e in 539ms, sequenceid=448, compaction requested=true 2024-11-13T03:50:35,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:35,990 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:35,991 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:35,993 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:35,993 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:35,993 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:35,993 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/fae4ae6242964e9fb4870276f8e4b430, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=53.7 K 2024-11-13T03:50:35,994 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:35,994 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:35,994 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:35,994 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2df1336737ba4af2bd49441f9efbba82, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=48.9 K 2024-11-13T03:50:35,995 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae4ae6242964e9fb4870276f8e4b430, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832850 2024-11-13T03:50:35,995 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2df1336737ba4af2bd49441f9efbba82, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832850 2024-11-13T03:50:35,995 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 777f54f1b3b64087bfa9d0f56f37194c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731469832970 2024-11-13T03:50:35,996 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d1ca08bc564a9b9e9de651119a1628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731469832970 2024-11-13T03:50:35,996 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 021c736bf1c34b5baf5869ca6cefc43c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731469833622 2024-11-13T03:50:35,996 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb2355ed42c482e9874434994bc2c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731469833622 2024-11-13T03:50:35,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ede1c4d1954d5ba42d3cdfb155abc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:35,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf09f694447643c28985a048b4334523, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:36,017 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:36,018 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/1026a8077ce8404c92d7a8385b597a68 is 50, key is test_row_0/B:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:36,021 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:36,022 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6c247b580af849f6abdd09c577587e65 is 50, key is test_row_0/A:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:36,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741927_1103 (size=13323) 2024-11-13T03:50:36,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741928_1104 (size=13323) 2024-11-13T03:50:36,068 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6c247b580af849f6abdd09c577587e65 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6c247b580af849f6abdd09c577587e65 2024-11-13T03:50:36,079 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 6c247b580af849f6abdd09c577587e65(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:36,079 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:36,079 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=12, startTime=1731469835990; duration=0sec 2024-11-13T03:50:36,080 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:36,080 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:36,080 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:50:36,082 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:50:36,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T03:50:36,083 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:36,083 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:36,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:50:36,083 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/4beefbe3185a470baec0323bde895890, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=48.9 K 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:36,084 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4beefbe3185a470baec0323bde895890, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731469832850 2024-11-13T03:50:36,084 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7172ee52ac74eb882bf72c22e524f51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731469832970 2024-11-13T03:50:36,085 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecddfbaeedcd4ff99114d14bc671ac67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731469833622 2024-11-13T03:50:36,085 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b02d8e6863740c982b4ff0a61ff8baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6e367688dc4c4acbbc418a440e01d6e5 is 50, key is test_row_0/A:col10/1731469835461/Put/seqid=0 2024-11-13T03:50:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:36,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:36,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741929_1105 (size=12301) 2024-11-13T03:50:36,125 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:36,126 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6f839c78b75a4028bc64b84217089e48 is 50, key is test_row_0/C:col10/1731469834364/Put/seqid=0 2024-11-13T03:50:36,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741930_1106 (size=13323) 2024-11-13T03:50:36,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469836126, max timestamp=1731469836148Num uniques 859; 2024-11-13T03:50:36,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:50:36,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469836126, max timestamp=1731469836148Num uniques 859; 2024-11-13T03:50:36,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:50:36,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469836126, max timestamp=1731469836148Num uniques 859; 2024-11-13T03:50:36,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:50:36,159 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/6f839c78b75a4028bc64b84217089e48 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6f839c78b75a4028bc64b84217089e48 2024-11-13T03:50:36,169 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 6f839c78b75a4028bc64b84217089e48(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:36,169 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:36,169 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=12, startTime=1731469835991; duration=0sec 2024-11-13T03:50:36,169 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:36,169 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:36,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469896183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469896190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469896191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469896297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469896297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469896307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:36,452 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/1026a8077ce8404c92d7a8385b597a68 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1026a8077ce8404c92d7a8385b597a68 2024-11-13T03:50:36,460 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 1026a8077ce8404c92d7a8385b597a68(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:36,460 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:36,460 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=12, startTime=1731469835990; duration=0sec 2024-11-13T03:50:36,460 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:36,461 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:36,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469896507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469896507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469896517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,521 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6e367688dc4c4acbbc418a440e01d6e5 2024-11-13T03:50:36,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ed794d4e3a63457bb7a6128bb488b4a7 is 50, key is test_row_0/B:col10/1731469835461/Put/seqid=0 2024-11-13T03:50:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741931_1107 (size=12301) 2024-11-13T03:50:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469896557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469896557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,564 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,564 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469896817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469896817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469896827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:36,936 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ed794d4e3a63457bb7a6128bb488b4a7 2024-11-13T03:50:36,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/195842d425604374ae32d1345ada3c0a is 50, key is test_row_0/C:col10/1731469835461/Put/seqid=0 2024-11-13T03:50:36,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741932_1108 (size=12301) 2024-11-13T03:50:36,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/195842d425604374ae32d1345ada3c0a 2024-11-13T03:50:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/6e367688dc4c4acbbc418a440e01d6e5 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5 2024-11-13T03:50:36,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5, entries=150, sequenceid=459, filesize=12.0 K 2024-11-13T03:50:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/ed794d4e3a63457bb7a6128bb488b4a7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7 2024-11-13T03:50:36,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7, entries=150, sequenceid=459, filesize=12.0 K 2024-11-13T03:50:36,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/195842d425604374ae32d1345ada3c0a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a 2024-11-13T03:50:36,975 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a, entries=150, sequenceid=459, filesize=12.0 K 2024-11-13T03:50:36,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 40c7504f29b68767a43cbb0fe4bac67e in 894ms, sequenceid=459, compaction requested=false 2024-11-13T03:50:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-13T03:50:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-13T03:50:36,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T03:50:36,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-13T03:50:36,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2130 sec 2024-11-13T03:50:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:37,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:37,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1a1182ced8a94abdb71a99b54ae7bf1c is 50, key is test_row_0/A:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:37,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741933_1109 (size=17181) 2024-11-13T03:50:37,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469897340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469897344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469897345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469897456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469897456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469897457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469897666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469897666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469897666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1a1182ced8a94abdb71a99b54ae7bf1c 2024-11-13T03:50:37,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/71e6c3914e624eca86abfd029f24f616 is 50, key is test_row_0/B:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:37,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741934_1110 (size=12301) 2024-11-13T03:50:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T03:50:37,918 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:37,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-13T03:50:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:37,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:37,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:37,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:37,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469897977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469897977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469897977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:38,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T03:50:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:38,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/71e6c3914e624eca86abfd029f24f616 2024-11-13T03:50:38,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/63792b170fae4e6cb75c47659934561b is 50, key is test_row_0/C:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:38,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741935_1111 (size=12301) 2024-11-13T03:50:38,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T03:50:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:38,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T03:50:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469898487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469898487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469898497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T03:50:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:38,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/63792b170fae4e6cb75c47659934561b 2024-11-13T03:50:38,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1a1182ced8a94abdb71a99b54ae7bf1c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c 2024-11-13T03:50:38,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c, entries=250, sequenceid=498, filesize=16.8 K 2024-11-13T03:50:38,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469898577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:38,581 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/71e6c3914e624eca86abfd029f24f616 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616 2024-11-13T03:50:38,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616, entries=150, sequenceid=498, filesize=12.0 K 2024-11-13T03:50:38,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/63792b170fae4e6cb75c47659934561b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b 2024-11-13T03:50:38,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469898587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:38,592 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:38,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b, entries=150, sequenceid=498, filesize=12.0 K 2024-11-13T03:50:38,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 40c7504f29b68767a43cbb0fe4bac67e in 1264ms, sequenceid=498, compaction requested=true 2024-11-13T03:50:38,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:38,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:38,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:38,596 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:38,597 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:38,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:38,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:38,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:38,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:38,598 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:38,598 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:38,598 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,598 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:38,598 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6c247b580af849f6abdd09c577587e65, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=41.8 K 2024-11-13T03:50:38,598 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:38,598 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,598 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1026a8077ce8404c92d7a8385b597a68, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.0 K 2024-11-13T03:50:38,602 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c247b580af849f6abdd09c577587e65, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:38,602 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1026a8077ce8404c92d7a8385b597a68, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:38,603 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ed794d4e3a63457bb7a6128bb488b4a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731469835451 2024-11-13T03:50:38,603 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e367688dc4c4acbbc418a440e01d6e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731469835451 2024-11-13T03:50:38,603 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e6c3914e624eca86abfd029f24f616, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836189 2024-11-13T03:50:38,604 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1182ced8a94abdb71a99b54ae7bf1c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836176 2024-11-13T03:50:38,614 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:38,615 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/174c9d65942543db868c7a493f4ab6c1 is 50, key is test_row_0/A:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:38,624 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:38,625 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/11b7b8e9bad747d3b78f5aedb37fcfb2 is 50, key is test_row_0/B:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:38,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741936_1112 (size=13425) 2024-11-13T03:50:38,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741937_1113 (size=13425) 2024-11-13T03:50:38,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/174c9d65942543db868c7a493f4ab6c1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/174c9d65942543db868c7a493f4ab6c1 2024-11-13T03:50:38,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/11b7b8e9bad747d3b78f5aedb37fcfb2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/11b7b8e9bad747d3b78f5aedb37fcfb2 2024-11-13T03:50:38,655 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 11b7b8e9bad747d3b78f5aedb37fcfb2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:38,655 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:38,655 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469838596; duration=0sec 2024-11-13T03:50:38,656 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:38,656 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:38,656 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:38,658 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:38,658 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:38,658 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 174c9d65942543db868c7a493f4ab6c1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:38,659 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:38,659 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,659 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469838596; duration=0sec 2024-11-13T03:50:38,659 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6f839c78b75a4028bc64b84217089e48, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.0 K 2024-11-13T03:50:38,659 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:38,659 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:38,659 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f839c78b75a4028bc64b84217089e48, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731469834355 2024-11-13T03:50:38,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 195842d425604374ae32d1345ada3c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731469835451 2024-11-13T03:50:38,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 63792b170fae4e6cb75c47659934561b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836189 2024-11-13T03:50:38,670 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:38,671 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e621540c23da4a8f83a964186ac87c8c is 50, key is test_row_0/C:col10/1731469837331/Put/seqid=0 2024-11-13T03:50:38,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T03:50:38,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:38,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0b2a31feb7fd4bd2adc2a1390450db26 is 50, key is test_row_1/A:col10/1731469837344/Put/seqid=0 2024-11-13T03:50:38,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741938_1114 (size=13425) 2024-11-13T03:50:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741939_1115 (size=9857) 2024-11-13T03:50:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:39,123 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0b2a31feb7fd4bd2adc2a1390450db26 2024-11-13T03:50:39,125 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e621540c23da4a8f83a964186ac87c8c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e621540c23da4a8f83a964186ac87c8c 2024-11-13T03:50:39,133 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into e621540c23da4a8f83a964186ac87c8c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a97d465dadf449c9742635ae1d77310 is 50, key is test_row_1/B:col10/1731469837344/Put/seqid=0 2024-11-13T03:50:39,133 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:39,133 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469838597; duration=0sec 2024-11-13T03:50:39,133 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:39,133 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741940_1116 (size=9857) 2024-11-13T03:50:39,150 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a97d465dadf449c9742635ae1d77310 2024-11-13T03:50:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/d1dc4851ebb741aeaf91ad61427fffcf is 50, key is test_row_1/C:col10/1731469837344/Put/seqid=0 2024-11-13T03:50:39,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741941_1117 (size=9857) 2024-11-13T03:50:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:39,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469839514, max timestamp=1731469839524Num uniques 859; 2024-11-13T03:50:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469839514, max timestamp=1731469839524Num uniques 859; 2024-11-13T03:50:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469839514, max timestamp=1731469839525Num uniques 859; 2024-11-13T03:50:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:39,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469899553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469839550, max timestamp=1731469839555Num uniques 1018; 2024-11-13T03:50:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:50:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469839550, max timestamp=1731469839555Num uniques 1018; 2024-11-13T03:50:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:50:39,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469839550, max timestamp=1731469839555Num uniques 1018; 2024-11-13T03:50:39,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:50:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469899594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731469839571, max timestamp=1731469839593Num uniques 1077; 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731469839571, max timestamp=1731469839593Num uniques 1077; 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731469839571, max timestamp=1731469839593Num uniques 1077; 2024-11-13T03:50:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-13T03:50:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,611 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/d1dc4851ebb741aeaf91ad61427fffcf 2024-11-13T03:50:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/0b2a31feb7fd4bd2adc2a1390450db26 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26 2024-11-13T03:50:39,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26, entries=100, sequenceid=508, filesize=9.6 K 2024-11-13T03:50:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a97d465dadf449c9742635ae1d77310 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310 2024-11-13T03:50:39,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310, entries=100, sequenceid=508, filesize=9.6 K 2024-11-13T03:50:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/d1dc4851ebb741aeaf91ad61427fffcf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf 2024-11-13T03:50:39,643 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf, entries=100, sequenceid=508, filesize=9.6 K 2024-11-13T03:50:39,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=107.34 KB/109920 for 40c7504f29b68767a43cbb0fe4bac67e in 957ms, sequenceid=508, compaction requested=false 2024-11-13T03:50:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-13T03:50:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-13T03:50:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:39,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=114.05 KB heapSize=309.47 KB 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:39,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T03:50:39,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7260 sec 2024-11-13T03:50:39,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7330 sec 2024-11-13T03:50:39,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/76f1cedcb38d4976a97b6bc4d8f05e4d is 50, key is test_row_0/A:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:39,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741942_1118 (size=17181) 2024-11-13T03:50:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469899682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469899683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469899707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469899796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469899796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469899917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469900006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469900007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/76f1cedcb38d4976a97b6bc4d8f05e4d 2024-11-13T03:50:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T03:50:40,068 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:40,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-13T03:50:40,070 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:40,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:40,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:40,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b3f88af386d947f8a45c65e65d36b611 is 50, key is test_row_0/B:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741943_1119 (size=12301) 2024-11-13T03:50:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:40,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:40,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469900226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469900316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469900316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:40,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b3f88af386d947f8a45c65e65d36b611 2024-11-13T03:50:40,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/fb20e7c0e0f147678e3b657558846e7b is 50, key is test_row_0/C:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:40,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741944_1120 (size=12301) 2024-11-13T03:50:40,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:40,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469900737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469900826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469900837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:40,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/fb20e7c0e0f147678e3b657558846e7b 2024-11-13T03:50:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/76f1cedcb38d4976a97b6bc4d8f05e4d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d 2024-11-13T03:50:40,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d, entries=250, sequenceid=574, filesize=16.8 K 2024-11-13T03:50:40,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b3f88af386d947f8a45c65e65d36b611 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611 2024-11-13T03:50:40,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611, entries=150, sequenceid=574, filesize=12.0 K 2024-11-13T03:50:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/fb20e7c0e0f147678e3b657558846e7b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b 2024-11-13T03:50:40,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b, entries=150, sequenceid=574, filesize=12.0 K 2024-11-13T03:50:40,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~309.42 KB/316848, currentSize=73.80 KB/75570 for 40c7504f29b68767a43cbb0fe4bac67e in 1276ms, sequenceid=574, compaction requested=true 2024-11-13T03:50:40,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:40,923 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:40,924 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:40,924 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40463 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:40,925 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:40,925 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,925 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:40,925 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:40,925 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/174c9d65942543db868c7a493f4ab6c1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=39.5 K 2024-11-13T03:50:40,925 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,925 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/11b7b8e9bad747d3b78f5aedb37fcfb2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=34.7 K 2024-11-13T03:50:40,926 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b7b8e9bad747d3b78f5aedb37fcfb2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836189 2024-11-13T03:50:40,926 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 174c9d65942543db868c7a493f4ab6c1, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836189 2024-11-13T03:50:40,927 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2a31feb7fd4bd2adc2a1390450db26, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731469837338 2024-11-13T03:50:40,927 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a97d465dadf449c9742635ae1d77310, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731469837338 2024-11-13T03:50:40,928 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f1cedcb38d4976a97b6bc4d8f05e4d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839621 2024-11-13T03:50:40,928 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f88af386d947f8a45c65e65d36b611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839625 2024-11-13T03:50:40,940 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:40,941 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/7635182eb3ac4188a5929d7c8ae237dc is 50, key is test_row_0/A:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:40,941 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:40,942 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b4662e48d62c4d28a311e6b327c2c64a is 50, key is test_row_0/B:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:40,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741946_1122 (size=13527) 2024-11-13T03:50:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741945_1121 (size=13527) 2024-11-13T03:50:40,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T03:50:40,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:40,988 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T03:50:40,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:40,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1eea9b8388a047a698bd18fbefb36b32 is 50, key is test_row_0/A:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:40,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741947_1123 (size=12301) 2024-11-13T03:50:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:41,365 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/b4662e48d62c4d28a311e6b327c2c64a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b4662e48d62c4d28a311e6b327c2c64a 2024-11-13T03:50:41,365 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/7635182eb3ac4188a5929d7c8ae237dc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7635182eb3ac4188a5929d7c8ae237dc 2024-11-13T03:50:41,373 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 7635182eb3ac4188a5929d7c8ae237dc(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:41,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:41,373 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469840923; duration=0sec 2024-11-13T03:50:41,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:41,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:41,373 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:41,375 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into b4662e48d62c4d28a311e6b327c2c64a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:41,375 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:41,375 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469840924; duration=0sec 2024-11-13T03:50:41,375 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:41,375 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:41,377 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:41,377 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:41,377 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:41,377 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e621540c23da4a8f83a964186ac87c8c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=34.7 K 2024-11-13T03:50:41,377 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e621540c23da4a8f83a964186ac87c8c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731469836189 2024-11-13T03:50:41,378 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d1dc4851ebb741aeaf91ad61427fffcf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731469837338 2024-11-13T03:50:41,378 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting fb20e7c0e0f147678e3b657558846e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839625 2024-11-13T03:50:41,393 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:41,394 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/0bbdd4a90e0b4e4398eb84da5fcf2204 is 50, key is test_row_0/C:col10/1731469839646/Put/seqid=0 2024-11-13T03:50:41,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1eea9b8388a047a698bd18fbefb36b32 2024-11-13T03:50:41,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741948_1124 (size=13527) 2024-11-13T03:50:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4497590a0bda4c86a54272e658d3abf0 is 50, key is test_row_0/B:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:41,413 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/0bbdd4a90e0b4e4398eb84da5fcf2204 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0bbdd4a90e0b4e4398eb84da5fcf2204 2024-11-13T03:50:41,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741949_1125 (size=12301) 2024-11-13T03:50:41,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4497590a0bda4c86a54272e658d3abf0 2024-11-13T03:50:41,424 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 0bbdd4a90e0b4e4398eb84da5fcf2204(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:41,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:41,424 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469840924; duration=0sec 2024-11-13T03:50:41,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:41,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/505a0dbd5f524364892cd7e0f7b69d34 is 50, key is test_row_0/C:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:41,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741950_1126 (size=12301) 2024-11-13T03:50:41,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/505a0dbd5f524364892cd7e0f7b69d34 2024-11-13T03:50:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/1eea9b8388a047a698bd18fbefb36b32 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32 2024-11-13T03:50:41,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32, entries=150, sequenceid=588, filesize=12.0 K 2024-11-13T03:50:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/4497590a0bda4c86a54272e658d3abf0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0 2024-11-13T03:50:41,464 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0, entries=150, sequenceid=588, filesize=12.0 K 2024-11-13T03:50:41,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/505a0dbd5f524364892cd7e0f7b69d34 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34 2024-11-13T03:50:41,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34, entries=150, sequenceid=588, filesize=12.0 K 2024-11-13T03:50:41,472 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 40c7504f29b68767a43cbb0fe4bac67e in 484ms, sequenceid=588, compaction requested=false 2024-11-13T03:50:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-13T03:50:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-13T03:50:41,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T03:50:41,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-13T03:50:41,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4080 sec 2024-11-13T03:50:41,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:41,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f392f11ded734d2eb4cd0820a9c4fd3b is 50, key is test_row_1/A:col10/1731469841770/Put/seqid=0 2024-11-13T03:50:41,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741951_1127 (size=12297) 2024-11-13T03:50:41,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f392f11ded734d2eb4cd0820a9c4fd3b 2024-11-13T03:50:41,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/7c71bc8f05984177972e07057a11642c is 50, key is test_row_1/B:col10/1731469841770/Put/seqid=0 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469841833, max timestamp=1731469841846Num uniques 859; 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469841833, max timestamp=1731469841846Num uniques 859; 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469841833, max timestamp=1731469841846Num uniques 859; 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:41,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:50:41,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741952_1128 (size=9857) 2024-11-13T03:50:41,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/7c71bc8f05984177972e07057a11642c 2024-11-13T03:50:41,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f40e848ad7b94268a3c87486f736ab4e is 50, key is test_row_1/C:col10/1731469841770/Put/seqid=0 2024-11-13T03:50:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741953_1129 (size=9857) 2024-11-13T03:50:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469901887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469901894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469901896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469902007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469902007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469902007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T03:50:42,217 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:42,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-13T03:50:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:42,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:42,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:42,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469902217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469902217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469902217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f40e848ad7b94268a3c87486f736ab4e 2024-11-13T03:50:42,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/f392f11ded734d2eb4cd0820a9c4fd3b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b 2024-11-13T03:50:42,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b, entries=150, sequenceid=601, filesize=12.0 K 2024-11-13T03:50:42,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/7c71bc8f05984177972e07057a11642c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c 2024-11-13T03:50:42,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c, entries=100, sequenceid=601, filesize=9.6 K 2024-11-13T03:50:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/f40e848ad7b94268a3c87486f736ab4e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e 2024-11-13T03:50:42,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e, entries=100, sequenceid=601, filesize=9.6 K 2024-11-13T03:50:42,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 40c7504f29b68767a43cbb0fe4bac67e in 541ms, sequenceid=601, compaction requested=true 2024-11-13T03:50:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:42,313 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:42,313 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:42,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:42,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:42,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:42,316 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38125 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:42,316 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:42,316 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:42,316 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7635182eb3ac4188a5929d7c8ae237dc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.2 K 2024-11-13T03:50:42,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:42,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:42,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:42,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b4662e48d62c4d28a311e6b327c2c64a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=34.8 K 2024-11-13T03:50:42,317 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7635182eb3ac4188a5929d7c8ae237dc, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839625 2024-11-13T03:50:42,318 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b4662e48d62c4d28a311e6b327c2c64a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839625 2024-11-13T03:50:42,318 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eea9b8388a047a698bd18fbefb36b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731469839657 2024-11-13T03:50:42,318 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4497590a0bda4c86a54272e658d3abf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731469839657 2024-11-13T03:50:42,318 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f392f11ded734d2eb4cd0820a9c4fd3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469841766 2024-11-13T03:50:42,319 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c71bc8f05984177972e07057a11642c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469841769 2024-11-13T03:50:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:42,332 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:42,332 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/4a21193bc7c94d60bd7316cae7c35ce2 is 50, key is test_row_0/A:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:42,336 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:42,337 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3f7826eeb32c41bcb7a09777a2672e40 is 50, key is test_row_0/B:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741954_1130 (size=13629) 2024-11-13T03:50:42,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741955_1131 (size=13629) 2024-11-13T03:50:42,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T03:50:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:42,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/a8d2c72f81a6481eb8cf1b337a3e264f is 50, key is test_row_0/A:col10/1731469841877/Put/seqid=0 2024-11-13T03:50:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741956_1132 (size=12301) 2024-11-13T03:50:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. as already flushing 2024-11-13T03:50:42,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469902533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469902534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:42,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469902535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33166 deadline: 1731469902596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,604 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731469902637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,640 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 40c7504f29b68767a43cbb0fe4bac67e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:50:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469902647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469902647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469902647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,745 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/4a21193bc7c94d60bd7316cae7c35ce2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4a21193bc7c94d60bd7316cae7c35ce2 2024-11-13T03:50:42,750 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 4a21193bc7c94d60bd7316cae7c35ce2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:42,750 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:42,750 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469842313; duration=0sec 2024-11-13T03:50:42,751 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:42,751 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:42,751 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:42,752 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:42,752 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:42,752 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:42,752 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0bbdd4a90e0b4e4398eb84da5fcf2204, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=34.8 K 2024-11-13T03:50:42,753 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbdd4a90e0b4e4398eb84da5fcf2204, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731469839625 2024-11-13T03:50:42,753 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/3f7826eeb32c41bcb7a09777a2672e40 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3f7826eeb32c41bcb7a09777a2672e40 2024-11-13T03:50:42,753 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 505a0dbd5f524364892cd7e0f7b69d34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731469839657 2024-11-13T03:50:42,754 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40e848ad7b94268a3c87486f736ab4e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469841769 2024-11-13T03:50:42,759 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 3f7826eeb32c41bcb7a09777a2672e40(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:42,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:42,759 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469842313; duration=0sec 2024-11-13T03:50:42,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:42,759 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:42,763 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:42,763 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e9573cb045ff46829eec822e8a8d1ad1 is 50, key is test_row_0/C:col10/1731469839678/Put/seqid=0 2024-11-13T03:50:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741957_1133 (size=13629) 2024-11-13T03:50:42,782 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/a8d2c72f81a6481eb8cf1b337a3e264f 2024-11-13T03:50:42,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb3fd0057e8d40d39713af802c3f8d24 is 50, key is test_row_0/B:col10/1731469841877/Put/seqid=0 2024-11-13T03:50:42,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741958_1134 (size=12301) 2024-11-13T03:50:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469902857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469902857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469902857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33178 deadline: 1731469903167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33208 deadline: 1731469903167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1731469903167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:50:43,173 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/e9573cb045ff46829eec822e8a8d1ad1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e9573cb045ff46829eec822e8a8d1ad1 2024-11-13T03:50:43,179 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into e9573cb045ff46829eec822e8a8d1ad1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:43,179 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:43,179 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469842314; duration=0sec 2024-11-13T03:50:43,179 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:43,179 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:43,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb3fd0057e8d40d39713af802c3f8d24 2024-11-13T03:50:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/09735ca4fbb54fa697eccb4ebb999e28 is 50, key is test_row_0/C:col10/1731469841877/Put/seqid=0 2024-11-13T03:50:43,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741959_1135 (size=12301) 2024-11-13T03:50:43,293 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T03:50:43,294 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,294 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T03:50:43,294 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,295 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T03:50:43,295 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,298 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T03:50:43,298 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,298 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,299 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:43,608 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/09735ca4fbb54fa697eccb4ebb999e28 2024-11-13T03:50:43,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/a8d2c72f81a6481eb8cf1b337a3e264f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f 2024-11-13T03:50:43,622 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f, entries=150, sequenceid=638, filesize=12.0 K 2024-11-13T03:50:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/eb3fd0057e8d40d39713af802c3f8d24 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24 2024-11-13T03:50:43,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24, entries=150, sequenceid=638, filesize=12.0 K 2024-11-13T03:50:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/09735ca4fbb54fa697eccb4ebb999e28 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28 2024-11-13T03:50:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28, entries=150, sequenceid=638, filesize=12.0 K 2024-11-13T03:50:43,635 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 40c7504f29b68767a43cbb0fe4bac67e in 1263ms, sequenceid=638, compaction requested=false 2024-11-13T03:50:43,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:43,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:43,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T03:50:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T03:50:43,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T03:50:43,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-13T03:50:43,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4220 sec 2024-11-13T03:50:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:43,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:50:43,678 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T03:50:43,678 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T03:50:43,678 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T03:50:43,678 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,678 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,678 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:43,678 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:43,678 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:43,679 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,679 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:43,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:43,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:43,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ed582b217f804042bf9598f21436fd08 is 50, key is test_row_0/A:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:43,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741960_1136 (size=12301) 2024-11-13T03:50:44,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ed582b217f804042bf9598f21436fd08 2024-11-13T03:50:44,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a3a7929f37b4738a86f76b169ee264b is 50, key is test_row_0/B:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:44,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741961_1137 (size=12301) 2024-11-13T03:50:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T03:50:44,369 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:44,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a3a7929f37b4738a86f76b169ee264b 2024-11-13T03:50:44,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/854765afdf884d2dad820df5f0a76aaa is 50, key is test_row_0/C:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:44,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741962_1138 (size=12301) 2024-11-13T03:50:44,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/854765afdf884d2dad820df5f0a76aaa 2024-11-13T03:50:44,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/ed582b217f804042bf9598f21436fd08 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08 2024-11-13T03:50:44,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08, entries=150, sequenceid=654, filesize=12.0 K 2024-11-13T03:50:44,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/2a3a7929f37b4738a86f76b169ee264b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b 2024-11-13T03:50:44,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b, entries=150, sequenceid=654, filesize=12.0 K 2024-11-13T03:50:44,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/854765afdf884d2dad820df5f0a76aaa as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa 2024-11-13T03:50:44,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa, entries=150, sequenceid=654, filesize=12.0 K 2024-11-13T03:50:44,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 40c7504f29b68767a43cbb0fe4bac67e in 1290ms, sequenceid=654, compaction requested=true 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:50:44,968 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:44,968 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40c7504f29b68767a43cbb0fe4bac67e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:50:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:44,969 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:44,969 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:44,969 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/B is initiating minor compaction (all files) 2024-11-13T03:50:44,969 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/A is initiating minor compaction (all files) 2024-11-13T03:50:44,969 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/B in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:44,970 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/A in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:44,970 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3f7826eeb32c41bcb7a09777a2672e40, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.3 K 2024-11-13T03:50:44,970 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4a21193bc7c94d60bd7316cae7c35ce2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.3 K 2024-11-13T03:50:44,970 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7826eeb32c41bcb7a09777a2672e40, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469839678 2024-11-13T03:50:44,970 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a21193bc7c94d60bd7316cae7c35ce2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469839678 2024-11-13T03:50:44,970 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting eb3fd0057e8d40d39713af802c3f8d24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731469841877 2024-11-13T03:50:44,970 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d2c72f81a6481eb8cf1b337a3e264f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731469841877 2024-11-13T03:50:44,971 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3a7929f37b4738a86f76b169ee264b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731469843677 2024-11-13T03:50:44,971 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed582b217f804042bf9598f21436fd08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731469843677 2024-11-13T03:50:44,980 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:44,980 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#A#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:44,981 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3d74be9c404440dc8b584665ee4e0fa9 is 50, key is test_row_0/A:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:44,981 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/67baebcaf0e54d27a19b48cc2acfef42 is 50, key is test_row_0/B:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:44,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741963_1139 (size=13731) 2024-11-13T03:50:44,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741964_1140 (size=13731) 2024-11-13T03:50:45,400 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/3d74be9c404440dc8b584665ee4e0fa9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3d74be9c404440dc8b584665ee4e0fa9 2024-11-13T03:50:45,400 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/67baebcaf0e54d27a19b48cc2acfef42 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67baebcaf0e54d27a19b48cc2acfef42 2024-11-13T03:50:45,406 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/B of 40c7504f29b68767a43cbb0fe4bac67e into 67baebcaf0e54d27a19b48cc2acfef42(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:45,406 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/A of 40c7504f29b68767a43cbb0fe4bac67e into 3d74be9c404440dc8b584665ee4e0fa9(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:45,407 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/A, priority=13, startTime=1731469844968; duration=0sec 2024-11-13T03:50:45,407 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/B, priority=13, startTime=1731469844968; duration=0sec 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:B 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:A 2024-11-13T03:50:45,407 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:50:45,408 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:50:45,408 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 40c7504f29b68767a43cbb0fe4bac67e/C is initiating minor compaction (all files) 2024-11-13T03:50:45,408 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40c7504f29b68767a43cbb0fe4bac67e/C in TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:45,408 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e9573cb045ff46829eec822e8a8d1ad1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp, totalSize=37.3 K 2024-11-13T03:50:45,408 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e9573cb045ff46829eec822e8a8d1ad1, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1731469839678 2024-11-13T03:50:45,409 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 09735ca4fbb54fa697eccb4ebb999e28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731469841877 2024-11-13T03:50:45,409 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 854765afdf884d2dad820df5f0a76aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1731469843677 2024-11-13T03:50:45,416 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40c7504f29b68767a43cbb0fe4bac67e#C#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:50:45,416 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/584584b649eb4e6e97ca577d5179cd6d is 50, key is test_row_0/C:col10/1731469843677/Put/seqid=0 2024-11-13T03:50:45,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741965_1141 (size=13731) 2024-11-13T03:50:45,561 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:50:45,832 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/584584b649eb4e6e97ca577d5179cd6d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/584584b649eb4e6e97ca577d5179cd6d 2024-11-13T03:50:45,838 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40c7504f29b68767a43cbb0fe4bac67e/C of 40c7504f29b68767a43cbb0fe4bac67e into 584584b649eb4e6e97ca577d5179cd6d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:50:45,838 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40c7504f29b68767a43cbb0fe4bac67e: 2024-11-13T03:50:45,838 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., storeName=40c7504f29b68767a43cbb0fe4bac67e/C, priority=13, startTime=1731469844968; duration=0sec 2024-11-13T03:50:45,838 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:50:45,838 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40c7504f29b68767a43cbb0fe4bac67e:C 2024-11-13T03:50:52,692 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T03:50:52,692 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:52,693 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,693 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:52,729 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T03:50:52,729 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:50:52,729 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,729 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3195 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2972 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:50:52,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1331 2024-11-13T03:50:52,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3992 rows 2024-11-13T03:50:52,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1334 2024-11-13T03:50:52,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3995 rows 2024-11-13T03:50:52,731 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:50:52,731 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:50:52,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:52,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:50:52,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:50:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:52,744 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469852744"}]},"ts":"1731469852744"} 2024-11-13T03:50:52,746 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:50:52,746 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:50:52,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:50:52,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, UNASSIGN}] 2024-11-13T03:50:52,752 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, UNASSIGN 2024-11-13T03:50:52,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=40c7504f29b68767a43cbb0fe4bac67e, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:52,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, UNASSIGN because future has completed 2024-11-13T03:50:52,755 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:50:52,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:52,913 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:52,914 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:50:52,915 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 40c7504f29b68767a43cbb0fe4bac67e, disabling compactions & flushes 2024-11-13T03:50:52,915 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:52,915 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:52,915 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. after waiting 0 ms 2024-11-13T03:50:52,915 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:52,916 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 40c7504f29b68767a43cbb0fe4bac67e 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T03:50:52,916 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=A 2024-11-13T03:50:52,916 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:52,916 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=B 2024-11-13T03:50:52,916 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:52,917 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40c7504f29b68767a43cbb0fe4bac67e, store=C 2024-11-13T03:50:52,917 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:52,924 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/b53b8be5595a46f2a7c48031c914a85a is 50, key is test_row_2/A:col10/1731469852727/Put/seqid=0 2024-11-13T03:50:52,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741966_1142 (size=7415) 2024-11-13T03:50:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:53,328 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/b53b8be5595a46f2a7c48031c914a85a 2024-11-13T03:50:53,336 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/979d7ac0bfc24925a32f716524a2438e is 50, key is test_row_2/B:col10/1731469852727/Put/seqid=0 2024-11-13T03:50:53,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741967_1143 (size=7415) 2024-11-13T03:50:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:53,742 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/979d7ac0bfc24925a32f716524a2438e 2024-11-13T03:50:53,756 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ec272387bace495f9ee72ed8a1ae8cb6 is 50, key is test_row_2/C:col10/1731469852727/Put/seqid=0 2024-11-13T03:50:53,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741968_1144 (size=7415) 2024-11-13T03:50:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:54,161 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ec272387bace495f9ee72ed8a1ae8cb6 2024-11-13T03:50:54,172 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/A/b53b8be5595a46f2a7c48031c914a85a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/b53b8be5595a46f2a7c48031c914a85a 2024-11-13T03:50:54,176 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/b53b8be5595a46f2a7c48031c914a85a, entries=50, sequenceid=662, filesize=7.2 K 2024-11-13T03:50:54,177 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/B/979d7ac0bfc24925a32f716524a2438e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/979d7ac0bfc24925a32f716524a2438e 2024-11-13T03:50:54,181 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/979d7ac0bfc24925a32f716524a2438e, entries=50, sequenceid=662, filesize=7.2 K 2024-11-13T03:50:54,181 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/.tmp/C/ec272387bace495f9ee72ed8a1ae8cb6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ec272387bace495f9ee72ed8a1ae8cb6 2024-11-13T03:50:54,186 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ec272387bace495f9ee72ed8a1ae8cb6, entries=50, sequenceid=662, filesize=7.2 K 2024-11-13T03:50:54,187 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 40c7504f29b68767a43cbb0fe4bac67e in 1272ms, sequenceid=662, compaction requested=false 2024-11-13T03:50:54,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/58fad009f44d447ba2a598a4353eca0d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6ca654c0a1c145e2818d98488fdf8664, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2387c15b1f24495a8ac0c24e9ec5cb5d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1357a88be9984268991fba074f6ec045, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/9af7db1a2ad04759823f01d5069e344a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f5c878bbfbe846af912049e457aa9e80, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf13417d19fd4bcea213fd613e0b55ac, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/fae4ae6242964e9fb4870276f8e4b430, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6c247b580af849f6abdd09c577587e65, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/174c9d65942543db868c7a493f4ab6c1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7635182eb3ac4188a5929d7c8ae237dc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4a21193bc7c94d60bd7316cae7c35ce2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08] to archive 2024-11-13T03:50:54,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:50:54,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bc21b9289ff2466f8fcd3c35c2dd1d5e 2024-11-13T03:50:54,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/dcad67208a93477e83a107dde72fcb0e 2024-11-13T03:50:54,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/85f2650309cb4ff692542648fc78f590 2024-11-13T03:50:54,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/58fad009f44d447ba2a598a4353eca0d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/58fad009f44d447ba2a598a4353eca0d 2024-11-13T03:50:54,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ce782f00f6414a909f2a3dae620eb465 2024-11-13T03:50:54,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/5ed64bab0c6d447287a1087daaee503c 2024-11-13T03:50:54,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6ca654c0a1c145e2818d98488fdf8664 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6ca654c0a1c145e2818d98488fdf8664 2024-11-13T03:50:54,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/72d18c685bd54f1dadc035efec8fae51 2024-11-13T03:50:54,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8bc6e4b08dbb434d8213f0c6287cd982 2024-11-13T03:50:54,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2615cd3c365e4163bd2d136f8b13a416 2024-11-13T03:50:54,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0f5571541ea34caca1523e24be0f999c 2024-11-13T03:50:54,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2387c15b1f24495a8ac0c24e9ec5cb5d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/2387c15b1f24495a8ac0c24e9ec5cb5d 2024-11-13T03:50:54,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3a55cdacef86418586619ef720e39759 2024-11-13T03:50:54,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f23130b541b7493896acee63ba320fc6 2024-11-13T03:50:54,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1357a88be9984268991fba074f6ec045 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1357a88be9984268991fba074f6ec045 2024-11-13T03:50:54,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/8b68a24c61a94e87906799cc9d606228 2024-11-13T03:50:54,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/9af7db1a2ad04759823f01d5069e344a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/9af7db1a2ad04759823f01d5069e344a 2024-11-13T03:50:54,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7c67071b8fa9430abf695643ac1cfa59 2024-11-13T03:50:54,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3f730ab0724249319d6d1b7c09e1dac0 2024-11-13T03:50:54,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4d3f1a79c30e41cf92ea7576db26e0c3 2024-11-13T03:50:54,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f5c878bbfbe846af912049e457aa9e80 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f5c878bbfbe846af912049e457aa9e80 2024-11-13T03:50:54,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3813797018f94e1991303a772181f579 2024-11-13T03:50:54,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/83ecf88d29244882a36fb1ade3a5446c 2024-11-13T03:50:54,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf13417d19fd4bcea213fd613e0b55ac to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf13417d19fd4bcea213fd613e0b55ac 2024-11-13T03:50:54,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/18886022534c4a909c6960bb2fe76074 2024-11-13T03:50:54,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0d53e180d666406180c556b712814b9c 2024-11-13T03:50:54,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/fae4ae6242964e9fb4870276f8e4b430 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/fae4ae6242964e9fb4870276f8e4b430 2024-11-13T03:50:54,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/777f54f1b3b64087bfa9d0f56f37194c 2024-11-13T03:50:54,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/eeb2355ed42c482e9874434994bc2c1a 2024-11-13T03:50:54,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/bf09f694447643c28985a048b4334523 2024-11-13T03:50:54,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6c247b580af849f6abdd09c577587e65 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6c247b580af849f6abdd09c577587e65 2024-11-13T03:50:54,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/6e367688dc4c4acbbc418a440e01d6e5 2024-11-13T03:50:54,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1a1182ced8a94abdb71a99b54ae7bf1c 2024-11-13T03:50:54,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/174c9d65942543db868c7a493f4ab6c1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/174c9d65942543db868c7a493f4ab6c1 2024-11-13T03:50:54,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/0b2a31feb7fd4bd2adc2a1390450db26 2024-11-13T03:50:54,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/76f1cedcb38d4976a97b6bc4d8f05e4d 2024-11-13T03:50:54,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7635182eb3ac4188a5929d7c8ae237dc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/7635182eb3ac4188a5929d7c8ae237dc 2024-11-13T03:50:54,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/1eea9b8388a047a698bd18fbefb36b32 2024-11-13T03:50:54,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4a21193bc7c94d60bd7316cae7c35ce2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/4a21193bc7c94d60bd7316cae7c35ce2 2024-11-13T03:50:54,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/f392f11ded734d2eb4cd0820a9c4fd3b 2024-11-13T03:50:54,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/a8d2c72f81a6481eb8cf1b337a3e264f 2024-11-13T03:50:54,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/ed582b217f804042bf9598f21436fd08 2024-11-13T03:50:54,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/471bc9ba50ef4fed8cef67d4a4bb4cd0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c92a4c7d2134408adaedd656be5fddd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4bcd78e6dc994f59b33570526f1bd71e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/e7f454b6df744dd5b41c10d715ffcf7d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/bdfe18f68a384d3fa32072f5f000f492, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0658edfce3144921be24c6ce5fc1b032, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a653f2f492f44017a8729c27cbb79ef9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2df1336737ba4af2bd49441f9efbba82, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1026a8077ce8404c92d7a8385b597a68, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/11b7b8e9bad747d3b78f5aedb37fcfb2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b4662e48d62c4d28a311e6b327c2c64a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3f7826eeb32c41bcb7a09777a2672e40, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b] to archive 2024-11-13T03:50:54,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:50:54,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/8cdb8bf5ff614019b44c8105a8210dc2 2024-11-13T03:50:54,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/68ed3646e1bf4283b39d3ccdd38dccd4 2024-11-13T03:50:54,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/471bc9ba50ef4fed8cef67d4a4bb4cd0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/471bc9ba50ef4fed8cef67d4a4bb4cd0 2024-11-13T03:50:54,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/cf6208e3963a4e5b9d65adb28b2c0bbe 2024-11-13T03:50:54,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb1b858c5ce847358e6093be68a48dea 2024-11-13T03:50:54,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/d1a7b8aacbce452eb3d385850c858dd6 2024-11-13T03:50:54,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c92a4c7d2134408adaedd656be5fddd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c92a4c7d2134408adaedd656be5fddd 2024-11-13T03:50:54,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4c6d0c57923b4486b33b04c1120582f6 2024-11-13T03:50:54,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0c1ae762fb42404db04c512f9fb8378d 2024-11-13T03:50:54,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1759e3a16c0d47689465ccc571a66594 2024-11-13T03:50:54,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4bcd78e6dc994f59b33570526f1bd71e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4bcd78e6dc994f59b33570526f1bd71e 2024-11-13T03:50:54,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/161bcbbbd901421aad5681081c2e3907 2024-11-13T03:50:54,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/332f89097e85461ea5aa80e8e8b3ac2a 2024-11-13T03:50:54,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/e7f454b6df744dd5b41c10d715ffcf7d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/e7f454b6df744dd5b41c10d715ffcf7d 2024-11-13T03:50:54,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/399ec091be6f4ec2afe78621a8ee3f9c 2024-11-13T03:50:54,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/adffa3af818848689ea34cbdcf13c1e6 2024-11-13T03:50:54,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/bdfe18f68a384d3fa32072f5f000f492 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/bdfe18f68a384d3fa32072f5f000f492 2024-11-13T03:50:54,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/08130fd61e3d4fbfb2ddcba9185cb4f2 2024-11-13T03:50:54,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4ea6d91b5f38473782ff121a974bc477 2024-11-13T03:50:54,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0658edfce3144921be24c6ce5fc1b032 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/0658edfce3144921be24c6ce5fc1b032 2024-11-13T03:50:54,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ff08f3fb8b024f15afda8c136009fbcf 2024-11-13T03:50:54,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/24b95047103e41e9a1cfcca48712ad89 2024-11-13T03:50:54,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a653f2f492f44017a8729c27cbb79ef9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a653f2f492f44017a8729c27cbb79ef9 2024-11-13T03:50:54,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3115ef70e29a4f7fb617ee6b99a23467 2024-11-13T03:50:54,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b37817606d014e8fa9f612acf4739492 2024-11-13T03:50:54,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2df1336737ba4af2bd49441f9efbba82 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2df1336737ba4af2bd49441f9efbba82 2024-11-13T03:50:54,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3d1d4b2e9b7c43bdb5e62f0efe9c42ae 2024-11-13T03:50:54,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/a3d1ca08bc564a9b9e9de651119a1628 2024-11-13T03:50:54,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/021c736bf1c34b5baf5869ca6cefc43c 2024-11-13T03:50:54,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1026a8077ce8404c92d7a8385b597a68 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/1026a8077ce8404c92d7a8385b597a68 2024-11-13T03:50:54,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67ede1c4d1954d5ba42d3cdfb155abc1 2024-11-13T03:50:54,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/ed794d4e3a63457bb7a6128bb488b4a7 2024-11-13T03:50:54,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/11b7b8e9bad747d3b78f5aedb37fcfb2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/11b7b8e9bad747d3b78f5aedb37fcfb2 2024-11-13T03:50:54,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/71e6c3914e624eca86abfd029f24f616 2024-11-13T03:50:54,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a97d465dadf449c9742635ae1d77310 2024-11-13T03:50:54,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b4662e48d62c4d28a311e6b327c2c64a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b4662e48d62c4d28a311e6b327c2c64a 2024-11-13T03:50:54,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/b3f88af386d947f8a45c65e65d36b611 2024-11-13T03:50:54,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/4497590a0bda4c86a54272e658d3abf0 2024-11-13T03:50:54,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3f7826eeb32c41bcb7a09777a2672e40 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/3f7826eeb32c41bcb7a09777a2672e40 2024-11-13T03:50:54,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/7c71bc8f05984177972e07057a11642c 2024-11-13T03:50:54,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/eb3fd0057e8d40d39713af802c3f8d24 2024-11-13T03:50:54,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/2a3a7929f37b4738a86f76b169ee264b 2024-11-13T03:50:54,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7f7d0e0ad7044661991fe1d86cd9b8c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7bc22652a2d74ac6a491ba357da7e995, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/b001bbfab9cc4225b1021aeb3cb9e16c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1bd6d35acd394939848dd3e823ca2e6c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/2f437c5263fb495aa0270749c98dea87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e87bf9a3aed947299e633d5078dc39b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6aee5e56e6374ce1bd9606d6464f2bec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/4beefbe3185a470baec0323bde895890, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6f839c78b75a4028bc64b84217089e48, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e621540c23da4a8f83a964186ac87c8c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0bbdd4a90e0b4e4398eb84da5fcf2204, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e9573cb045ff46829eec822e8a8d1ad1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa] to archive 2024-11-13T03:50:54,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:50:54,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/58f25670d7fb44e4975e93d2a9005c95 2024-11-13T03:50:54,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/007b9eea1c67498895e1b3327e6ea5d8 2024-11-13T03:50:54,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7f7d0e0ad7044661991fe1d86cd9b8c9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7f7d0e0ad7044661991fe1d86cd9b8c9 2024-11-13T03:50:54,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/8dbc18def97f4178bd5f52800467b34c 2024-11-13T03:50:54,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/861538a8d6ba4fd9bbc68cfa0b9f11c2 2024-11-13T03:50:54,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/86a49e79da00449f818815c41f305d9f 2024-11-13T03:50:54,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7bc22652a2d74ac6a491ba357da7e995 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7bc22652a2d74ac6a491ba357da7e995 2024-11-13T03:50:54,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/94cc237205e145b38e51d5a7259d3f7a 2024-11-13T03:50:54,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6c0f77b412af4f5f9d54f342df1a0f2e 2024-11-13T03:50:54,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/da2c153b737f49558a42eaf9c55bb768 2024-11-13T03:50:54,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/b001bbfab9cc4225b1021aeb3cb9e16c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/b001bbfab9cc4225b1021aeb3cb9e16c 2024-11-13T03:50:54,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a07f473a3d9a40c19cdae17e4797d50c 2024-11-13T03:50:54,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1dae856e3acc4fd188105fe9a043f5ad 2024-11-13T03:50:54,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1bd6d35acd394939848dd3e823ca2e6c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/1bd6d35acd394939848dd3e823ca2e6c 2024-11-13T03:50:54,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ac82ac332b25450dba673055c2a3d49c 2024-11-13T03:50:54,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/bcfdd2ad8e2b4422949ab49327bc03f5 2024-11-13T03:50:54,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/2f437c5263fb495aa0270749c98dea87 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/2f437c5263fb495aa0270749c98dea87 2024-11-13T03:50:54,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a14b4822c0d5472d934d121adc0369c3 2024-11-13T03:50:54,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/285458940d764260b80288fda83f5e39 2024-11-13T03:50:54,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e87bf9a3aed947299e633d5078dc39b8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e87bf9a3aed947299e633d5078dc39b8 2024-11-13T03:50:54,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e21cb3ba0c2f4ea88608812556318c15 2024-11-13T03:50:54,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/be4ae693ca844b4fa5a29628f7ef0f33 2024-11-13T03:50:54,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6aee5e56e6374ce1bd9606d6464f2bec to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6aee5e56e6374ce1bd9606d6464f2bec 2024-11-13T03:50:54,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/a95f1d327d9647f686207221f81ac3c4 2024-11-13T03:50:54,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0dece04818c44d869f66199e53d303ff 2024-11-13T03:50:54,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/4beefbe3185a470baec0323bde895890 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/4beefbe3185a470baec0323bde895890 2024-11-13T03:50:54,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/98938c26571d430ebe73a045b9042624 2024-11-13T03:50:54,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f7172ee52ac74eb882bf72c22e524f51 2024-11-13T03:50:54,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ecddfbaeedcd4ff99114d14bc671ac67 2024-11-13T03:50:54,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6f839c78b75a4028bc64b84217089e48 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/6f839c78b75a4028bc64b84217089e48 2024-11-13T03:50:54,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/7b02d8e6863740c982b4ff0a61ff8baf 2024-11-13T03:50:54,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/195842d425604374ae32d1345ada3c0a 2024-11-13T03:50:54,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e621540c23da4a8f83a964186ac87c8c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e621540c23da4a8f83a964186ac87c8c 2024-11-13T03:50:54,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/63792b170fae4e6cb75c47659934561b 2024-11-13T03:50:54,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/d1dc4851ebb741aeaf91ad61427fffcf 2024-11-13T03:50:54,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0bbdd4a90e0b4e4398eb84da5fcf2204 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/0bbdd4a90e0b4e4398eb84da5fcf2204 2024-11-13T03:50:54,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/fb20e7c0e0f147678e3b657558846e7b 2024-11-13T03:50:54,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/505a0dbd5f524364892cd7e0f7b69d34 2024-11-13T03:50:54,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e9573cb045ff46829eec822e8a8d1ad1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/e9573cb045ff46829eec822e8a8d1ad1 2024-11-13T03:50:54,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/f40e848ad7b94268a3c87486f736ab4e 2024-11-13T03:50:54,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/09735ca4fbb54fa697eccb4ebb999e28 2024-11-13T03:50:54,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/854765afdf884d2dad820df5f0a76aaa 2024-11-13T03:50:54,364 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/recovered.edits/665.seqid, newMaxSeqId=665, maxSeqId=1 2024-11-13T03:50:54,365 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. 2024-11-13T03:50:54,365 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 40c7504f29b68767a43cbb0fe4bac67e: Waiting for close lock at 1731469852915Running coprocessor pre-close hooks at 1731469852915Disabling compacts and flushes for region at 1731469852915Disabling writes for close at 1731469852915Obtaining lock to block concurrent updates at 1731469852916 (+1 ms)Preparing flush snapshotting stores in 40c7504f29b68767a43cbb0fe4bac67e at 1731469852916Finished memstore snapshotting TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731469852917 (+1 ms)Flushing stores of TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e. at 1731469852918 (+1 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/A: creating writer at 1731469852918Flushing 40c7504f29b68767a43cbb0fe4bac67e/A: appending metadata at 1731469852923 (+5 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/A: closing flushed file at 1731469852923Flushing 40c7504f29b68767a43cbb0fe4bac67e/B: creating writer at 1731469853333 (+410 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/B: appending metadata at 1731469853336 (+3 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/B: closing flushed file at 1731469853336Flushing 40c7504f29b68767a43cbb0fe4bac67e/C: creating writer at 1731469853752 (+416 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/C: appending metadata at 1731469853755 (+3 ms)Flushing 40c7504f29b68767a43cbb0fe4bac67e/C: closing flushed file at 1731469853755Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d745a98: reopening flushed file at 1731469854171 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@af13c3b: reopening flushed file at 1731469854176 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e1002ed: reopening flushed file at 1731469854181 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 40c7504f29b68767a43cbb0fe4bac67e in 1272ms, sequenceid=662, compaction requested=false at 1731469854187 (+6 ms)Writing region close event to WAL at 1731469854361 (+174 ms)Running coprocessor post-close hooks at 1731469854365 (+4 ms)Closed at 1731469854365 2024-11-13T03:50:54,368 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:54,369 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=40c7504f29b68767a43cbb0fe4bac67e, regionState=CLOSED 2024-11-13T03:50:54,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:50:54,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T03:50:54,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 40c7504f29b68767a43cbb0fe4bac67e, server=ac1a51731247,42123,1731469817167 in 1.6150 sec 2024-11-13T03:50:54,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T03:50:54,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40c7504f29b68767a43cbb0fe4bac67e, UNASSIGN in 1.6220 sec 2024-11-13T03:50:54,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T03:50:54,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6280 sec 2024-11-13T03:50:54,377 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469854377"}]},"ts":"1731469854377"} 2024-11-13T03:50:54,379 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:50:54,379 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:50:54,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6420 sec 2024-11-13T03:50:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T03:50:54,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:54,890 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:50:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,894 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T03:50:54,896 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:54,903 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/recovered.edits] 2024-11-13T03:50:54,906 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3d74be9c404440dc8b584665ee4e0fa9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/3d74be9c404440dc8b584665ee4e0fa9 2024-11-13T03:50:54,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/b53b8be5595a46f2a7c48031c914a85a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/A/b53b8be5595a46f2a7c48031c914a85a 2024-11-13T03:50:54,909 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67baebcaf0e54d27a19b48cc2acfef42 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/67baebcaf0e54d27a19b48cc2acfef42 2024-11-13T03:50:54,911 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/979d7ac0bfc24925a32f716524a2438e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/B/979d7ac0bfc24925a32f716524a2438e 2024-11-13T03:50:54,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/584584b649eb4e6e97ca577d5179cd6d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/584584b649eb4e6e97ca577d5179cd6d 2024-11-13T03:50:54,914 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ec272387bace495f9ee72ed8a1ae8cb6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/C/ec272387bace495f9ee72ed8a1ae8cb6 2024-11-13T03:50:54,916 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/recovered.edits/665.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e/recovered.edits/665.seqid 2024-11-13T03:50:54,916 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/40c7504f29b68767a43cbb0fe4bac67e 2024-11-13T03:50:54,916 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:50:54,921 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T03:50:54,930 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:50:54,961 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:50:54,962 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,962 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:50:54,962 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731469854962"}]},"ts":"9223372036854775807"} 2024-11-13T03:50:54,966 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:50:54,966 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 40c7504f29b68767a43cbb0fe4bac67e, NAME => 'TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:50:54,966 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:50:54,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731469854966"}]},"ts":"9223372036854775807"} 2024-11-13T03:50:54,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:50:54,970 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:54,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-13T03:50:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T03:50:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:50:54,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:55,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ac1a51731247:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ac1a51731247:42123-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-36274116_22 at /127.0.0.1:41162 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 206) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8364 (was 8760) 2024-11-13T03:50:55,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=8364 2024-11-13T03:50:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:50:55,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:50:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:55,021 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:50:55,021 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:55,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-13T03:50:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:55,022 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:50:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741969_1145 (size=1061) 2024-11-13T03:50:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:55,430 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:50:55,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741970_1146 (size=53) 2024-11-13T03:50:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:55,841 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:55,841 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5bf9df9b5e7eb4d19bdee05e09462bbe, disabling compactions & flushes 2024-11-13T03:50:55,841 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:55,841 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:55,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. after waiting 0 ms 2024-11-13T03:50:55,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:55,842 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:55,842 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: Waiting for close lock at 1731469855841Disabling compacts and flushes for region at 1731469855841Disabling writes for close at 1731469855842 (+1 ms)Writing region close event to WAL at 1731469855842Closed at 1731469855842 2024-11-13T03:50:55,846 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:50:55,847 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469855846"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469855846"}]},"ts":"1731469855846"} 2024-11-13T03:50:55,850 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:50:55,852 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:50:55,852 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469855852"}]},"ts":"1731469855852"} 2024-11-13T03:50:55,883 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:50:55,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, ASSIGN}] 2024-11-13T03:50:55,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, ASSIGN 2024-11-13T03:50:55,885 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:50:56,036 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:56,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, ASSIGN because future has completed 2024-11-13T03:50:56,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:56,206 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:56,207 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:50:56,208 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,208 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:56,209 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,209 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,212 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,214 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:56,214 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName A 2024-11-13T03:50:56,215 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:56,215 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:56,216 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,217 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:56,217 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName B 2024-11-13T03:50:56,217 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:56,218 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:56,218 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,219 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:56,219 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName C 2024-11-13T03:50:56,220 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:56,220 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:56,220 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:56,221 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,221 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,222 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,223 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,223 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,224 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:50:56,226 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,229 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:50:56,230 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5bf9df9b5e7eb4d19bdee05e09462bbe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64184420, jitterRate=-0.043577611446380615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:50:56,230 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:56,231 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: Running coprocessor pre-open hook at 1731469856209Writing region info on filesystem at 1731469856209Initializing all the Stores at 1731469856211 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469856211Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469856211Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469856211Cleaning up temporary data from old regions at 1731469856223 (+12 ms)Running coprocessor post-open hooks at 1731469856230 (+7 ms)Region opened successfully at 1731469856231 (+1 ms) 2024-11-13T03:50:56,232 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., pid=34, masterSystemTime=1731469856197 2024-11-13T03:50:56,235 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:56,235 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:56,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:56,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:50:56,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-13T03:50:56,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 in 197 msec 2024-11-13T03:50:56,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T03:50:56,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, ASSIGN in 358 msec 2024-11-13T03:50:56,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:50:56,245 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469856245"}]},"ts":"1731469856245"} 2024-11-13T03:50:56,247 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:50:56,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:50:56,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2290 sec 2024-11-13T03:50:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T03:50:57,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:57,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:57,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:57,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:57,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60560153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:57,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:57,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:57,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:57,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:50:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:50:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@296fe578 2024-11-13T03:50:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:50:57,198 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:50:57,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:50:57,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:50:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T03:50:57,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741971_1147 (size=1097) 2024-11-13T03:50:57,623 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T03:50:57,623 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T03:50:57,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:50:57,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE}] 2024-11-13T03:50:57,633 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE 2024-11-13T03:50:57,633 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:57,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE because future has completed 2024-11-13T03:50:57,635 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:50:57,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:57,788 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:57,788 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:50:57,788 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5bf9df9b5e7eb4d19bdee05e09462bbe, disabling compactions & flushes 2024-11-13T03:50:57,788 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:57,788 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:57,788 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. after waiting 0 ms 2024-11-13T03:50:57,788 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:57,792 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T03:50:57,793 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:57,793 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: Waiting for close lock at 1731469857788Running coprocessor pre-close hooks at 1731469857788Disabling compacts and flushes for region at 1731469857788Disabling writes for close at 1731469857788Writing region close event to WAL at 1731469857789 (+1 ms)Running coprocessor post-close hooks at 1731469857793 (+4 ms)Closed at 1731469857793 2024-11-13T03:50:57,793 WARN [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5bf9df9b5e7eb4d19bdee05e09462bbe to self. 2024-11-13T03:50:57,795 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:57,795 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=CLOSED 2024-11-13T03:50:57,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:50:57,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-13T03:50:57,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 in 163 msec 2024-11-13T03:50:57,800 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE; state=CLOSED, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=true 2024-11-13T03:50:57,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:57,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE because future has completed 2024-11-13T03:50:57,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:50:58,116 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,116 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:50:58,117 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,117 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:50:58,117 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,117 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,121 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,122 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:58,127 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName A 2024-11-13T03:50:58,130 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:58,130 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:58,131 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,131 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:58,132 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName B 2024-11-13T03:50:58,132 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:58,132 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:58,132 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,133 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:50:58,133 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf9df9b5e7eb4d19bdee05e09462bbe columnFamilyName C 2024-11-13T03:50:58,133 DEBUG [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:58,133 INFO [StoreOpener-5bf9df9b5e7eb4d19bdee05e09462bbe-1 {}] regionserver.HStore(327): Store=5bf9df9b5e7eb4d19bdee05e09462bbe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:50:58,134 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,134 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,134 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,135 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,136 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,136 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,137 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:50:58,138 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,138 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5bf9df9b5e7eb4d19bdee05e09462bbe; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71114833, jitterRate=0.0596935898065567}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:50:58,138 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,139 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: Running coprocessor pre-open hook at 1731469858117Writing region info on filesystem at 1731469858117Initializing all the Stores at 1731469858119 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469858119Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469858119Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469858119Cleaning up temporary data from old regions at 1731469858136 (+17 ms)Running coprocessor post-open hooks at 1731469858138 (+2 ms)Region opened successfully at 1731469858139 (+1 ms) 2024-11-13T03:50:58,140 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., pid=39, masterSystemTime=1731469858108 2024-11-13T03:50:58,142 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,142 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,142 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=OPEN, openSeqNum=5, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:50:58,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-13T03:50:58,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 in 190 msec 2024-11-13T03:50:58,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T03:50:58,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, REOPEN/MOVE in 514 msec 2024-11-13T03:50:58,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-13T03:50:58,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-11-13T03:50:58,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-13T03:50:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-13T03:50:58,157 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:50:58,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca3dd57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,160 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,162 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec17a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,165 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62573cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667013ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692061b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5489715e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,172 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cf49a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,174 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8134d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,176 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6d153a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280f6764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,180 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa74f11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0905f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,183 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c3e521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,184 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d65f25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:50:58,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:50:58,186 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:50:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:50:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:50:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6ab570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:50:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:50:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:50:58,188 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:50:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:50:58,189 DEBUG [Thread-729 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,189 DEBUG [Thread-738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,189 DEBUG [Thread-725 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,190 DEBUG [Thread-721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,190 DEBUG [Thread-732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,190 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:50:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,190 DEBUG [Thread-734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,191 DEBUG [Thread-727 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,191 DEBUG [Thread-723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T03:50:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,191 DEBUG [Thread-736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:50:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:50:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:50:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:50:58,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:50:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:50:58,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:50:58,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:50:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:50:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:50:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:50:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:50:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:50:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:50:58,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111389f9f58d0671469eaec7631ed2a5fdc3_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469858200/Put/seqid=0 2024-11-13T03:50:58,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469858239, max timestamp=1731469858274Num uniques 859; 2024-11-13T03:50:58,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:58,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469858239, max timestamp=1731469858274Num uniques 859; 2024-11-13T03:50:58,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:58,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731469858239, max timestamp=1731469858274Num uniques 859; 2024-11-13T03:50:58,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T03:50:58,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741972_1148 (size=14594) 2024-11-13T03:50:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:50:58,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469918330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469918337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469918339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469918341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469918343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469918448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469918458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469918458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469918458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469918458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:50:58,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469918657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469918668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469918668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469918669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469918669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,684 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:58,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111389f9f58d0671469eaec7631ed2a5fdc3_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389f9f58d0671469eaec7631ed2a5fdc3_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:58,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1c194f84e5914b1b8f5c2dc345b7eba0, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:50:58,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1c194f84e5914b1b8f5c2dc345b7eba0 is 175, key is test_row_0/A:col10/1731469858200/Put/seqid=0 2024-11-13T03:50:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741973_1149 (size=39549) 2024-11-13T03:50:58,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1c194f84e5914b1b8f5c2dc345b7eba0 2024-11-13T03:50:58,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 is 50, key is test_row_0/B:col10/1731469858200/Put/seqid=0 2024-11-13T03:50:58,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741974_1150 (size=12001) 2024-11-13T03:50:58,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 2024-11-13T03:50:58,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:50:58,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/bb49611b708d4babba207c40009953f8 is 50, key is test_row_0/C:col10/1731469858200/Put/seqid=0 2024-11-13T03:50:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741975_1151 (size=12001) 2024-11-13T03:50:58,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:58,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469918977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469918977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469918978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469918978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469918978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:59,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:50:59,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/bb49611b708d4babba207c40009953f8 2024-11-13T03:50:59,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1c194f84e5914b1b8f5c2dc345b7eba0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0 2024-11-13T03:50:59,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0, entries=200, sequenceid=15, filesize=38.6 K 2024-11-13T03:50:59,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 2024-11-13T03:50:59,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T03:50:59,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/bb49611b708d4babba207c40009953f8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8 2024-11-13T03:50:59,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T03:50:59,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1052ms, sequenceid=15, compaction requested=false 2024-11-13T03:50:59,263 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T03:50:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:50:59,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T03:50:59,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:50:59,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-13T03:50:59,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:50:59,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:59,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:50:59,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:59,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:50:59,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:50:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b3f51c31614c48e6adc7c88b187f050c_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469858282/Put/seqid=0 2024-11-13T03:50:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:50:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741976_1152 (size=12154) 2024-11-13T03:50:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:50:59,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b3f51c31614c48e6adc7c88b187f050c_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3f51c31614c48e6adc7c88b187f050c_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:59,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1df3b5f6ad1a48d59b22d2ad51b968db, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:50:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1df3b5f6ad1a48d59b22d2ad51b968db is 175, key is test_row_0/A:col10/1731469858282/Put/seqid=0 2024-11-13T03:50:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741977_1153 (size=30955) 2024-11-13T03:50:59,358 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1df3b5f6ad1a48d59b22d2ad51b968db 2024-11-13T03:50:59,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/b43192a9a094431e92de13d4bbc0f2df is 50, key is test_row_0/B:col10/1731469858282/Put/seqid=0 2024-11-13T03:50:59,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741978_1154 (size=12001) 2024-11-13T03:50:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:50:59,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:50:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469919504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469919505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469919506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469919508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469919509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469919617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469919617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469919617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469919617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469919627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,782 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/b43192a9a094431e92de13d4bbc0f2df 2024-11-13T03:50:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/5b515ca09ac4467d8fc4972c32f2b136 is 50, key is test_row_0/C:col10/1731469858282/Put/seqid=0 2024-11-13T03:50:59,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741979_1155 (size=12001) 2024-11-13T03:50:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469919827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469919828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469919829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469919829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:50:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469919837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:50:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:50:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:50:59,915 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:51:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469920137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469920137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469920137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469920137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469920147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,199 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/5b515ca09ac4467d8fc4972c32f2b136 2024-11-13T03:51:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/1df3b5f6ad1a48d59b22d2ad51b968db as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db 2024-11-13T03:51:00,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T03:51:00,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/b43192a9a094431e92de13d4bbc0f2df as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df 2024-11-13T03:51:00,226 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T03:51:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/5b515ca09ac4467d8fc4972c32f2b136 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136 2024-11-13T03:51:00,240 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T03:51:00,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 965ms, sequenceid=49, compaction requested=false 2024-11-13T03:51:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T03:51:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T03:51:00,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T03:51:00,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0510 sec 2024-11-13T03:51:00,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0580 sec 2024-11-13T03:51:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T03:51:00,340 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:00,340 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T03:51:00,342 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:00,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:00,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:00,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T03:51:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:00,499 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:00,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111361d29f614bdf4e768b24050961758eaa_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469859500/Put/seqid=0 2024-11-13T03:51:00,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741980_1156 (size=12154) 2024-11-13T03:51:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:00,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:00,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469920674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469920674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469920676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469920676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469860665, max timestamp=1731469860679Num uniques 859; 2024-11-13T03:51:00,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469860665, max timestamp=1731469860679Num uniques 859; 2024-11-13T03:51:00,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:00,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469860665, max timestamp=1731469860679Num uniques 859; 2024-11-13T03:51:00,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469920759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469920787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469920787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469920787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469920787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,858 INFO [master/ac1a51731247:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T03:51:00,858 INFO [master/ac1a51731247:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T03:51:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469920867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:00,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:00,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111361d29f614bdf4e768b24050961758eaa_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361d29f614bdf4e768b24050961758eaa_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:00,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c16e6e0806f441baafd4d23b90084cf3, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:00,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c16e6e0806f441baafd4d23b90084cf3 is 175, key is test_row_0/A:col10/1731469859500/Put/seqid=0 2024-11-13T03:51:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:00,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741981_1157 (size=30955) 2024-11-13T03:51:01,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469920997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469920997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469920998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469920998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469921076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469921306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469921306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469921307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469921307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,380 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c16e6e0806f441baafd4d23b90084cf3 2024-11-13T03:51:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/de7bca5cbaf448968711cc8e3d36d948 is 50, key is test_row_0/B:col10/1731469859500/Put/seqid=0 2024-11-13T03:51:01,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469921387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741982_1158 (size=12001) 2024-11-13T03:51:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:01,798 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/de7bca5cbaf448968711cc8e3d36d948 2024-11-13T03:51:01,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/e9b6bf14954c40f98f6aa4dcbec5a068 is 50, key is test_row_0/C:col10/1731469859500/Put/seqid=0 2024-11-13T03:51:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741983_1159 (size=12001) 2024-11-13T03:51:01,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/e9b6bf14954c40f98f6aa4dcbec5a068 2024-11-13T03:51:01,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c16e6e0806f441baafd4d23b90084cf3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3 2024-11-13T03:51:01,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469921817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469921817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,822 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3, entries=150, sequenceid=60, filesize=30.2 K 2024-11-13T03:51:01,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469921817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469921818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/de7bca5cbaf448968711cc8e3d36d948 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:01,828 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948, entries=150, sequenceid=60, filesize=11.7 K 2024-11-13T03:51:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/e9b6bf14954c40f98f6aa4dcbec5a068 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068 2024-11-13T03:51:01,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068, entries=150, sequenceid=60, filesize=11.7 K 2024-11-13T03:51:01,837 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1339ms, sequenceid=60, compaction requested=true 2024-11-13T03:51:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T03:51:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T03:51:01,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T03:51:01,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-11-13T03:51:01,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5010 sec 2024-11-13T03:51:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:01,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:01,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111310baed3a89a24beb864e78a72e4b33b3_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741984_1160 (size=14594) 2024-11-13T03:51:01,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:01,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111310baed3a89a24beb864e78a72e4b33b3_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310baed3a89a24beb864e78a72e4b33b3_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:01,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/176f2713c0144390aff8414f64ae3367, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:01,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/176f2713c0144390aff8414f64ae3367 is 175, key is test_row_0/A:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:01,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741985_1161 (size=39549) 2024-11-13T03:51:01,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469921944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469922057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469922267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/176f2713c0144390aff8414f64ae3367 2024-11-13T03:51:02,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/f5d934f603cd4182b485f0b9d66dd66e is 50, key is test_row_0/B:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:02,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741986_1162 (size=12001) 2024-11-13T03:51:02,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/f5d934f603cd4182b485f0b9d66dd66e 2024-11-13T03:51:02,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4c4c34764f094a6dbcec3323af02cc53 is 50, key is test_row_0/C:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:02,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741987_1163 (size=12001) 2024-11-13T03:51:02,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4c4c34764f094a6dbcec3323af02cc53 2024-11-13T03:51:02,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/176f2713c0144390aff8414f64ae3367 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367 2024-11-13T03:51:02,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367, entries=200, sequenceid=96, filesize=38.6 K 2024-11-13T03:51:02,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/f5d934f603cd4182b485f0b9d66dd66e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e 2024-11-13T03:51:02,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T03:51:02,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4c4c34764f094a6dbcec3323af02cc53 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53 2024-11-13T03:51:02,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T03:51:02,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 523ms, sequenceid=96, compaction requested=true 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:02,423 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:02,423 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:02,429 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:02,429 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:02,429 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,430 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=137.7 K 2024-11-13T03:51:02,430 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,430 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367] 2024-11-13T03:51:02,431 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:02,431 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c194f84e5914b1b8f5c2dc345b7eba0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469858200 2024-11-13T03:51:02,431 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:02,431 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,431 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=46.9 K 2024-11-13T03:51:02,431 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df3b5f6ad1a48d59b22d2ad51b968db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469858282 2024-11-13T03:51:02,431 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 320076e8ad8c4a96a2e2ed9cc3c8ceb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469858200 2024-11-13T03:51:02,432 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b43192a9a094431e92de13d4bbc0f2df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469858282 2024-11-13T03:51:02,432 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c16e6e0806f441baafd4d23b90084cf3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731469859500 2024-11-13T03:51:02,432 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176f2713c0144390aff8414f64ae3367, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:02,432 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting de7bca5cbaf448968711cc8e3d36d948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731469859500 2024-11-13T03:51:02,433 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d934f603cd4182b485f0b9d66dd66e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:02,446 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,453 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:02,454 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/0e4169076dff4b3bbaab87a5bf3de5de is 50, key is test_row_0/B:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:02,466 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132e990e31108e405595c40230b395f618_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,472 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132e990e31108e405595c40230b395f618_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,472 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132e990e31108e405595c40230b395f618_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741988_1164 (size=12139) 2024-11-13T03:51:02,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741989_1165 (size=4469) 2024-11-13T03:51:02,483 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#144 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:02,484 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c2f32568b4364292ba39b8e62ab420b0 is 175, key is test_row_0/A:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:02,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741990_1166 (size=31093) 2024-11-13T03:51:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T03:51:02,497 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:02,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:02,498 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c2f32568b4364292ba39b8e62ab420b0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0 2024-11-13T03:51:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T03:51:02,500 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:02,501 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:02,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:02,515 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into c2f32568b4364292ba39b8e62ab420b0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:02,515 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:02,515 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=12, startTime=1731469862423; duration=0sec 2024-11-13T03:51:02,516 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:02,516 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:02,516 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:02,521 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:02,521 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:02,521 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,521 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=46.9 K 2024-11-13T03:51:02,522 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb49611b708d4babba207c40009953f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469858200 2024-11-13T03:51:02,522 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b515ca09ac4467d8fc4972c32f2b136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469858282 2024-11-13T03:51:02,523 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9b6bf14954c40f98f6aa4dcbec5a068, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731469859500 2024-11-13T03:51:02,523 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4c34764f094a6dbcec3323af02cc53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:02,537 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:02,538 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/04ce30d3724347b78f0b3dddc7b0c8fc is 50, key is test_row_0/C:col10/1731469860736/Put/seqid=0 2024-11-13T03:51:02,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741991_1167 (size=12139) 2024-11-13T03:51:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:02,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:02,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f47f4a8aac74f879420ac617acc27b5_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469861905/Put/seqid=0 2024-11-13T03:51:02,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741992_1168 (size=14594) 2024-11-13T03:51:02,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:02,637 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f47f4a8aac74f879420ac617acc27b5_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f47f4a8aac74f879420ac617acc27b5_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:02,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/983899c2ff304e828a8ba292196b1750, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/983899c2ff304e828a8ba292196b1750 is 175, key is test_row_0/A:col10/1731469861905/Put/seqid=0 2024-11-13T03:51:02,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741993_1169 (size=39549) 2024-11-13T03:51:02,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/983899c2ff304e828a8ba292196b1750 2024-11-13T03:51:02,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469862646, max timestamp=1731469862682Num uniques 859; 2024-11-13T03:51:02,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469862646, max timestamp=1731469862682Num uniques 859; 2024-11-13T03:51:02,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:02,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:02,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469862646, max timestamp=1731469862682Num uniques 859; 2024-11-13T03:51:02,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:02,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ca34b601247944ca901288bbc7c4bddd is 50, key is test_row_0/B:col10/1731469861905/Put/seqid=0 2024-11-13T03:51:02,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741994_1170 (size=12001) 2024-11-13T03:51:02,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ca34b601247944ca901288bbc7c4bddd 2024-11-13T03:51:02,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/58b9b70985db4ae386f509bfd8663019 is 50, key is test_row_0/C:col10/1731469861905/Put/seqid=0 2024-11-13T03:51:02,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741995_1171 (size=12001) 2024-11-13T03:51:02,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/58b9b70985db4ae386f509bfd8663019 2024-11-13T03:51:02,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/983899c2ff304e828a8ba292196b1750 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750 2024-11-13T03:51:02,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750, entries=200, sequenceid=108, filesize=38.6 K 2024-11-13T03:51:02,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ca34b601247944ca901288bbc7c4bddd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd 2024-11-13T03:51:02,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T03:51:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/58b9b70985db4ae386f509bfd8663019 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019 2024-11-13T03:51:02,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T03:51:02,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=100.63 KB/103050 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 194ms, sequenceid=108, compaction requested=false 2024-11-13T03:51:02,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:02,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=107.34 KB heapSize=300.61 KB 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:02,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138d812ccf0ae545bdaede39bbf79f386a_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:02,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:02,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:02,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:02,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741996_1172 (size=17234) 2024-11-13T03:51:02,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:02,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138d812ccf0ae545bdaede39bbf79f386a_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138d812ccf0ae545bdaede39bbf79f386a_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:02,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/6507973b5d8b4f24b62095428d2b318b, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:02,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/6507973b5d8b4f24b62095428d2b318b is 175, key is test_row_0/A:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469922837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469922837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,846 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:02,846 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469922840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469922848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469922848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741997_1173 (size=48339) 2024-11-13T03:51:02,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/6507973b5d8b4f24b62095428d2b318b 2024-11-13T03:51:02,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/e5f3bf81fe3541bbbee9bdcb61276ca1 is 50, key is test_row_0/B:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:02,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/0e4169076dff4b3bbaab87a5bf3de5de as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/0e4169076dff4b3bbaab87a5bf3de5de 2024-11-13T03:51:02,890 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into 0e4169076dff4b3bbaab87a5bf3de5de(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:02,890 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:02,891 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=12, startTime=1731469862423; duration=0sec 2024-11-13T03:51:02,891 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:02,891 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:02,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741998_1174 (size=12151) 2024-11-13T03:51:02,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/e5f3bf81fe3541bbbee9bdcb61276ca1 2024-11-13T03:51:02,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d4c02986756043748f5302b201a160c9 is 50, key is test_row_0/C:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:02,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741999_1175 (size=12151) 2024-11-13T03:51:02,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:02,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:02,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469922957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469922957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469922957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:02,973 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/04ce30d3724347b78f0b3dddc7b0c8fc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/04ce30d3724347b78f0b3dddc7b0c8fc 2024-11-13T03:51:02,981 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into 04ce30d3724347b78f0b3dddc7b0c8fc(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:02,981 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:02,981 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=12, startTime=1731469862423; duration=0sec 2024-11-13T03:51:02,981 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:02,981 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:03,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:03,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:03,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469923177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469923177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469923178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:03,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d4c02986756043748f5302b201a160c9 2024-11-13T03:51:03,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/6507973b5d8b4f24b62095428d2b318b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b 2024-11-13T03:51:03,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b, entries=250, sequenceid=140, filesize=47.2 K 2024-11-13T03:51:03,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/e5f3bf81fe3541bbbee9bdcb61276ca1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1 2024-11-13T03:51:03,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1, entries=150, sequenceid=140, filesize=11.9 K 2024-11-13T03:51:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d4c02986756043748f5302b201a160c9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9 2024-11-13T03:51:03,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9, entries=150, sequenceid=140, filesize=11.9 K 2024-11-13T03:51:03,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.56 KB/307776, currentSize=73.80 KB/75570 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 569ms, sequenceid=140, compaction requested=true 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:03,348 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:03,348 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:03,349 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:03,349 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:03,349 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,349 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/0e4169076dff4b3bbaab87a5bf3de5de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.4 K 2024-11-13T03:51:03,349 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:03,349 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:03,349 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,349 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=116.2 K 2024-11-13T03:51:03,350 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b] 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4169076dff4b3bbaab87a5bf3de5de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2f32568b4364292ba39b8e62ab420b0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ca34b601247944ca901288bbc7c4bddd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731469861905 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983899c2ff304e828a8ba292196b1750, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731469861905 2024-11-13T03:51:03,350 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f3bf81fe3541bbbee9bdcb61276ca1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862749 2024-11-13T03:51:03,351 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6507973b5d8b4f24b62095428d2b318b, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862682 2024-11-13T03:51:03,358 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:03,361 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:03,362 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/6f28f305d5b9439d8c2c47999a69e8a9 is 50, key is test_row_0/B:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:03,364 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fb23df8902b54d9887242e48e481f227_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:03,367 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fb23df8902b54d9887242e48e481f227_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:03,368 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fb23df8902b54d9887242e48e481f227_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:03,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742000_1176 (size=12391) 2024-11-13T03:51:03,388 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/6f28f305d5b9439d8c2c47999a69e8a9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/6f28f305d5b9439d8c2c47999a69e8a9 2024-11-13T03:51:03,394 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into 6f28f305d5b9439d8c2c47999a69e8a9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:03,394 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:03,394 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=13, startTime=1731469863348; duration=0sec 2024-11-13T03:51:03,394 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:03,394 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:03,395 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:03,396 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:03,397 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:03,398 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,398 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/04ce30d3724347b78f0b3dddc7b0c8fc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.4 K 2024-11-13T03:51:03,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742001_1177 (size=4469) 2024-11-13T03:51:03,400 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ce30d3724347b78f0b3dddc7b0c8fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731469860736 2024-11-13T03:51:03,401 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b9b70985db4ae386f509bfd8663019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731469861905 2024-11-13T03:51:03,401 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c02986756043748f5302b201a160c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862749 2024-11-13T03:51:03,409 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:03,409 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d1fb15bebf0a41449aa9481271ca6b81 is 50, key is test_row_0/C:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:03,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,421 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:03,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742002_1178 (size=12391) 2024-11-13T03:51:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c9de1705f50f487a962375c4a868fa85_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469862826/Put/seqid=0 2024-11-13T03:51:03,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742003_1179 (size=12304) 2024-11-13T03:51:03,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:03,444 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c9de1705f50f487a962375c4a868fa85_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9de1705f50f487a962375c4a868fa85_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:03,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/53914615f14b40f89c83d4d8e8ce9b4c, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/53914615f14b40f89c83d4d8e8ce9b4c is 175, key is test_row_0/A:col10/1731469862826/Put/seqid=0 2024-11-13T03:51:03,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742004_1180 (size=31105) 2024-11-13T03:51:03,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/53914615f14b40f89c83d4d8e8ce9b4c 2024-11-13T03:51:03,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/526182bb61fb4fa2a505c9627292ddec is 50, key is test_row_0/B:col10/1731469862826/Put/seqid=0 2024-11-13T03:51:03,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742005_1181 (size=12151) 2024-11-13T03:51:03,474 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/526182bb61fb4fa2a505c9627292ddec 2024-11-13T03:51:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/2dab0acddfe44a52bfccb6ff88a5fd88 is 50, key is test_row_0/C:col10/1731469862826/Put/seqid=0 2024-11-13T03:51:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:03,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:03,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742006_1182 (size=12151) 2024-11-13T03:51:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469923518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469923519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469923519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469923627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469923627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469923627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:03,801 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:03,802 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/81bcdb4c695347e8aa3cef4bbedcba2e is 175, key is test_row_0/A:col10/1731469862765/Put/seqid=0 2024-11-13T03:51:03,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742007_1183 (size=31345) 2024-11-13T03:51:03,834 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d1fb15bebf0a41449aa9481271ca6b81 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d1fb15bebf0a41449aa9481271ca6b81 2024-11-13T03:51:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469923837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469923837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469923837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:03,843 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into d1fb15bebf0a41449aa9481271ca6b81(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:03,843 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:03,843 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=13, startTime=1731469863348; duration=0sec 2024-11-13T03:51:03,843 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:03,843 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:03,896 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/2dab0acddfe44a52bfccb6ff88a5fd88 2024-11-13T03:51:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/53914615f14b40f89c83d4d8e8ce9b4c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c 2024-11-13T03:51:03,905 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c, entries=150, sequenceid=157, filesize=30.4 K 2024-11-13T03:51:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/526182bb61fb4fa2a505c9627292ddec as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec 2024-11-13T03:51:03,910 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T03:51:03,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/2dab0acddfe44a52bfccb6ff88a5fd88 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88 2024-11-13T03:51:03,916 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T03:51:03,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 495ms, sequenceid=157, compaction requested=false 2024-11-13T03:51:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T03:51:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T03:51:03,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T03:51:03,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-13T03:51:03,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4220 sec 2024-11-13T03:51:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:04,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:04,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed877a62903046de9032e321a3ee8538_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469924160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469924162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469924163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742008_1184 (size=14794) 2024-11-13T03:51:04,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:04,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed877a62903046de9032e321a3ee8538_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed877a62903046de9032e321a3ee8538_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:04,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/80b3045fa9a541c68fc2bfabbc7ff3fe, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:04,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/80b3045fa9a541c68fc2bfabbc7ff3fe is 175, key is test_row_0/A:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:04,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742009_1185 (size=39749) 2024-11-13T03:51:04,219 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/81bcdb4c695347e8aa3cef4bbedcba2e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e 2024-11-13T03:51:04,224 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into 81bcdb4c695347e8aa3cef4bbedcba2e(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:04,224 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:04,224 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=13, startTime=1731469863348; duration=0sec 2024-11-13T03:51:04,224 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:04,224 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:04,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469924267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469924277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469924277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469924477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469924487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469924487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/80b3045fa9a541c68fc2bfabbc7ff3fe 2024-11-13T03:51:04,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5bd78907b3f146b0ba5d83d9d1db991a is 50, key is test_row_0/B:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:04,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742010_1186 (size=12151) 2024-11-13T03:51:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T03:51:04,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:04,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T03:51:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:04,649 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:04,650 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:04,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:04,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469924787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:04,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469924797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469924797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469924867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,867 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469924877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:04,879 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:04,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:04,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:05,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5bd78907b3f146b0ba5d83d9d1db991a 2024-11-13T03:51:05,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/577699881ae843d98f1bf7efa0deee86 is 50, key is test_row_0/C:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:05,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742011_1187 (size=12151) 2024-11-13T03:51:05,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:05,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469925296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:05,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469925317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469925317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:05,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:05,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/577699881ae843d98f1bf7efa0deee86 2024-11-13T03:51:05,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/80b3045fa9a541c68fc2bfabbc7ff3fe as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe 2024-11-13T03:51:05,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe, entries=200, sequenceid=177, filesize=38.8 K 2024-11-13T03:51:05,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5bd78907b3f146b0ba5d83d9d1db991a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a 2024-11-13T03:51:05,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T03:51:05,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/577699881ae843d98f1bf7efa0deee86 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86 2024-11-13T03:51:05,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T03:51:05,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1288ms, sequenceid=177, compaction requested=true 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:05,437 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:05,437 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:05,438 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:05,438 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:05,438 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:05,438 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:05,438 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,438 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,438 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/6f28f305d5b9439d8c2c47999a69e8a9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.8 K 2024-11-13T03:51:05,438 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=99.8 K 2024-11-13T03:51:05,438 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,439 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe] 2024-11-13T03:51:05,439 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f28f305d5b9439d8c2c47999a69e8a9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862749 2024-11-13T03:51:05,439 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81bcdb4c695347e8aa3cef4bbedcba2e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862749 2024-11-13T03:51:05,440 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 526182bb61fb4fa2a505c9627292ddec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731469862826 2024-11-13T03:51:05,440 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53914615f14b40f89c83d4d8e8ce9b4c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731469862826 2024-11-13T03:51:05,440 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd78907b3f146b0ba5d83d9d1db991a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:05,440 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b3045fa9a541c68fc2bfabbc7ff3fe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:05,451 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:05,453 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:05,454 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/1196e91c2bd64c2699ce74f2c6b041dd is 50, key is test_row_0/B:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:05,454 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132efc5ed0e54e4f9e9b6399c0445ecd94_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:05,457 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132efc5ed0e54e4f9e9b6399c0445ecd94_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:05,457 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132efc5ed0e54e4f9e9b6399c0445ecd94_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:05,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742013_1189 (size=4469) 2024-11-13T03:51:05,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742012_1188 (size=12493) 2024-11-13T03:51:05,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,567 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133a83c5e6f608475dabb7a8d35e3ca905_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469864159/Put/seqid=0 2024-11-13T03:51:05,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742014_1190 (size=12304) 2024-11-13T03:51:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:05,873 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:05,873 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/3479dc60ba88481cb07ab47a8750c93c is 175, key is test_row_0/A:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:05,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742015_1191 (size=31447) 2024-11-13T03:51:05,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/1196e91c2bd64c2699ce74f2c6b041dd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1196e91c2bd64c2699ce74f2c6b041dd 2024-11-13T03:51:05,889 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into 1196e91c2bd64c2699ce74f2c6b041dd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:05,889 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:05,889 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=13, startTime=1731469865437; duration=0sec 2024-11-13T03:51:05,889 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:05,889 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:05,889 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:05,890 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:05,890 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:05,890 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:05,890 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d1fb15bebf0a41449aa9481271ca6b81, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.8 K 2024-11-13T03:51:05,891 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fb15bebf0a41449aa9481271ca6b81, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731469862749 2024-11-13T03:51:05,891 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dab0acddfe44a52bfccb6ff88a5fd88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731469862826 2024-11-13T03:51:05,891 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 577699881ae843d98f1bf7efa0deee86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:05,899 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:05,899 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/054c7fccbdd1455788de41a80f3af29e is 50, key is test_row_0/C:col10/1731469864148/Put/seqid=0 2024-11-13T03:51:05,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742016_1192 (size=12493) 2024-11-13T03:51:05,911 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/054c7fccbdd1455788de41a80f3af29e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/054c7fccbdd1455788de41a80f3af29e 2024-11-13T03:51:05,919 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into 054c7fccbdd1455788de41a80f3af29e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:05,919 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:05,919 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=13, startTime=1731469865437; duration=0sec 2024-11-13T03:51:05,919 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:05,919 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:05,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:06,024 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133a83c5e6f608475dabb7a8d35e3ca905_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a83c5e6f608475dabb7a8d35e3ca905_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:06,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/55c2fc905e914087897043716a85fe84, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:06,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/55c2fc905e914087897043716a85fe84 is 175, key is test_row_0/A:col10/1731469864159/Put/seqid=0 2024-11-13T03:51:06,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742017_1193 (size=31105) 2024-11-13T03:51:06,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/55c2fc905e914087897043716a85fe84 2024-11-13T03:51:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ec4c1c5e2ab9403889b00cc4756d9f74 is 50, key is test_row_0/B:col10/1731469864159/Put/seqid=0 2024-11-13T03:51:06,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742018_1194 (size=12151) 2024-11-13T03:51:06,282 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/3479dc60ba88481cb07ab47a8750c93c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c 2024-11-13T03:51:06,287 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into 3479dc60ba88481cb07ab47a8750c93c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:06,287 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:06,287 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=13, startTime=1731469865437; duration=0sec 2024-11-13T03:51:06,287 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:06,287 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469926339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469926344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469926344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469926446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,459 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ec4c1c5e2ab9403889b00cc4756d9f74 2024-11-13T03:51:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469926457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469926457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/de6560bfeb3e415f98a400361d3ce426 is 50, key is test_row_0/C:col10/1731469864159/Put/seqid=0 2024-11-13T03:51:06,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742019_1195 (size=12151) 2024-11-13T03:51:06,482 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/de6560bfeb3e415f98a400361d3ce426 2024-11-13T03:51:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/55c2fc905e914087897043716a85fe84 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84 2024-11-13T03:51:06,493 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84, entries=150, sequenceid=191, filesize=30.4 K 2024-11-13T03:51:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/ec4c1c5e2ab9403889b00cc4756d9f74 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74 2024-11-13T03:51:06,499 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T03:51:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/de6560bfeb3e415f98a400361d3ce426 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426 2024-11-13T03:51:06,504 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T03:51:06,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 938ms, sequenceid=191, compaction requested=false 2024-11-13T03:51:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T03:51:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T03:51:06,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T03:51:06,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-13T03:51:06,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8620 sec 2024-11-13T03:51:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:06,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:06,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:06,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f52164c189344d78c9c0dec5698ce51_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:06,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742020_1196 (size=14794) 2024-11-13T03:51:06,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469926702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469926703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469926708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T03:51:06,807 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:06,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T03:51:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:06,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:06,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:06,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469926817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469926817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469926817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:06,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:06,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469927027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469927027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469927027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:07,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f52164c189344d78c9c0dec5698ce51_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f52164c189344d78c9c0dec5698ce51_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:07,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/ce67a15634b3451baa6622581ecafdcc, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:07,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/ce67a15634b3451baa6622581ecafdcc is 175, key is test_row_0/A:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:07,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742021_1197 (size=39749) 2024-11-13T03:51:07,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:07,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469927336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469927337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469927337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:07,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/ce67a15634b3451baa6622581ecafdcc 2024-11-13T03:51:07,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/d4b83b28e1874ff8ac134c178fd269de is 50, key is test_row_0/B:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:07,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742022_1198 (size=12151) 2024-11-13T03:51:07,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469927846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469927846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469927846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:07,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:07,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:07,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:07,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:07,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/d4b83b28e1874ff8ac134c178fd269de 2024-11-13T03:51:07,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4bb38c6815cf4dc1b5968e722d125855 is 50, key is test_row_0/C:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:07,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742023_1199 (size=12151) 2024-11-13T03:51:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:08,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:08,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:08,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:08,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4bb38c6815cf4dc1b5968e722d125855 2024-11-13T03:51:08,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/ce67a15634b3451baa6622581ecafdcc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc 2024-11-13T03:51:08,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc, entries=200, sequenceid=214, filesize=38.8 K 2024-11-13T03:51:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/d4b83b28e1874ff8ac134c178fd269de as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de 2024-11-13T03:51:08,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de, entries=150, sequenceid=214, filesize=11.9 K 2024-11-13T03:51:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/4bb38c6815cf4dc1b5968e722d125855 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855 2024-11-13T03:51:08,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855, entries=150, sequenceid=214, filesize=11.9 K 2024-11-13T03:51:08,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T03:51:08,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1678ms, sequenceid=214, compaction requested=true 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:08,337 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:08,337 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:08,337 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:08,338 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:08,338 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:08,338 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:08,338 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:08,338 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,338 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,338 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1196e91c2bd64c2699ce74f2c6b041dd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.9 K 2024-11-13T03:51:08,338 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=99.9 K 2024-11-13T03:51:08,339 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,339 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc] 2024-11-13T03:51:08,339 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1196e91c2bd64c2699ce74f2c6b041dd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:08,339 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3479dc60ba88481cb07ab47a8750c93c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:08,339 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4c1c5e2ab9403889b00cc4756d9f74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469864157 2024-11-13T03:51:08,340 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b83b28e1874ff8ac134c178fd269de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866342 2024-11-13T03:51:08,342 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c2fc905e914087897043716a85fe84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469864157 2024-11-13T03:51:08,342 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce67a15634b3451baa6622581ecafdcc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866335 2024-11-13T03:51:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f14e75fc0aa44a4b9f47ad361dc65765_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469866666/Put/seqid=0 2024-11-13T03:51:08,350 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742024_1200 (size=12304) 2024-11-13T03:51:08,357 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:08,357 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130e7cb91b18454d21b58ed9cc997070f0_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:08,357 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/d9d2c3e59a5546d5a98b5d0f450b5c74 is 50, key is test_row_0/B:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:08,359 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130e7cb91b18454d21b58ed9cc997070f0_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:08,360 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130e7cb91b18454d21b58ed9cc997070f0_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:08,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742025_1201 (size=12595) 2024-11-13T03:51:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742026_1202 (size=4469) 2024-11-13T03:51:08,367 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#172 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:08,367 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/b6a58d6dcf65443586016dbad2531cc9 is 175, key is test_row_0/A:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:08,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742027_1203 (size=31549) 2024-11-13T03:51:08,383 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/b6a58d6dcf65443586016dbad2531cc9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9 2024-11-13T03:51:08,388 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into b6a58d6dcf65443586016dbad2531cc9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:08,388 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:08,388 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=13, startTime=1731469868337; duration=0sec 2024-11-13T03:51:08,388 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:08,388 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:08,388 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:08,389 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:08,389 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:08,390 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:08,390 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/054c7fccbdd1455788de41a80f3af29e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=35.9 K 2024-11-13T03:51:08,390 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054c7fccbdd1455788de41a80f3af29e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731469863503 2024-11-13T03:51:08,390 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6560bfeb3e415f98a400361d3ce426, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469864157 2024-11-13T03:51:08,391 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb38c6815cf4dc1b5968e722d125855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866342 2024-11-13T03:51:08,400 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:08,400 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/82dbb29156164b1c89d30d0c22b9de96 is 50, key is test_row_0/C:col10/1731469866342/Put/seqid=0 2024-11-13T03:51:08,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742028_1204 (size=12595) 2024-11-13T03:51:08,422 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/82dbb29156164b1c89d30d0c22b9de96 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/82dbb29156164b1c89d30d0c22b9de96 2024-11-13T03:51:08,427 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into 82dbb29156164b1c89d30d0c22b9de96(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:08,427 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:08,427 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=13, startTime=1731469868337; duration=0sec 2024-11-13T03:51:08,428 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:08,428 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:08,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f14e75fc0aa44a4b9f47ad361dc65765_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f14e75fc0aa44a4b9f47ad361dc65765_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:08,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/de22017ae9954192a82b50006a161a3e, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:08,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/de22017ae9954192a82b50006a161a3e is 175, key is test_row_0/A:col10/1731469866666/Put/seqid=0 2024-11-13T03:51:08,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742029_1205 (size=31105) 2024-11-13T03:51:08,770 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/d9d2c3e59a5546d5a98b5d0f450b5c74 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d9d2c3e59a5546d5a98b5d0f450b5c74 2024-11-13T03:51:08,775 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into d9d2c3e59a5546d5a98b5d0f450b5c74(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:08,775 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:08,775 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=13, startTime=1731469868337; duration=0sec 2024-11-13T03:51:08,776 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:08,776 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469928893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469928893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1731469928896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469928895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731469928897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:08,899 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:08,900 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 5bf9df9b5e7eb4d19bdee05e09462bbe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:09,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469929007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469929007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469929007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/de22017ae9954192a82b50006a161a3e 2024-11-13T03:51:09,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/2c022ba9fae94c04848ba95c378fadd4 is 50, key is test_row_0/B:col10/1731469866666/Put/seqid=0 2024-11-13T03:51:09,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742030_1206 (size=12151) 2024-11-13T03:51:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469929217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469929217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469929217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469929526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469929526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469929526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:09,580 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/2c022ba9fae94c04848ba95c378fadd4 2024-11-13T03:51:09,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/b4bdd5df847c4c76b12a5ba9cce50e31 is 50, key is test_row_0/C:col10/1731469866666/Put/seqid=0 2024-11-13T03:51:09,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742031_1207 (size=12151) 2024-11-13T03:51:09,995 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/b4bdd5df847c4c76b12a5ba9cce50e31 2024-11-13T03:51:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/de22017ae9954192a82b50006a161a3e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e 2024-11-13T03:51:10,005 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e, entries=150, sequenceid=226, filesize=30.4 K 2024-11-13T03:51:10,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/2c022ba9fae94c04848ba95c378fadd4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4 2024-11-13T03:51:10,011 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T03:51:10,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/b4bdd5df847c4c76b12a5ba9cce50e31 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31 2024-11-13T03:51:10,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T03:51:10,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1681ms, sequenceid=226, compaction requested=false 2024-11-13T03:51:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T03:51:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T03:51:10,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T03:51:10,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2090 sec 2024-11-13T03:51:10,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.2140 sec 2024-11-13T03:51:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:10,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:10,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111307b7a8744dfe4537a7212bc50c96c701_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:10,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469930049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469930056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469930057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742032_1208 (size=12304) 2024-11-13T03:51:10,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469930166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469930167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469930167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469930376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469930376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469930376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:10,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111307b7a8744dfe4537a7212bc50c96c701_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111307b7a8744dfe4537a7212bc50c96c701_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:10,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/8b01f029d33041149075fdf71f48e7e2, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:10,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/8b01f029d33041149075fdf71f48e7e2 is 175, key is test_row_0/A:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:10,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742033_1209 (size=31105) 2024-11-13T03:51:10,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469930687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469930687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469930687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:10,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/8b01f029d33041149075fdf71f48e7e2 2024-11-13T03:51:10,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5209e681759e432284c61204c8387a71 is 50, key is test_row_0/B:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742034_1210 (size=12151) 2024-11-13T03:51:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T03:51:10,967 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:10,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-13T03:51:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:10,969 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:10,970 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:10,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:11,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T03:51:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469931206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469931206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469931207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:11,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T03:51:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:11,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5209e681759e432284c61204c8387a71 2024-11-13T03:51:11,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/1d0582d409604333ab639eb3956c4c3d is 50, key is test_row_0/C:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742035_1211 (size=12151) 2024-11-13T03:51:11,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T03:51:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:11,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T03:51:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:11,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:11,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/1d0582d409604333ab639eb3956c4c3d 2024-11-13T03:51:11,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/8b01f029d33041149075fdf71f48e7e2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2 2024-11-13T03:51:11,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2, entries=150, sequenceid=251, filesize=30.4 K 2024-11-13T03:51:11,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/5209e681759e432284c61204c8387a71 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71 2024-11-13T03:51:11,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T03:51:11,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/1d0582d409604333ab639eb3956c4c3d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d 2024-11-13T03:51:11,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T03:51:11,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1684ms, sequenceid=251, compaction requested=true 2024-11-13T03:51:11,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:11,724 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:11,724 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:11,725 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,725 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,725 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=91.6 K 2024-11-13T03:51:11,725 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d9d2c3e59a5546d5a98b5d0f450b5c74, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=36.0 K 2024-11-13T03:51:11,725 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2] 2024-11-13T03:51:11,725 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d9d2c3e59a5546d5a98b5d0f450b5c74, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866342 2024-11-13T03:51:11,726 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6a58d6dcf65443586016dbad2531cc9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866342 2024-11-13T03:51:11,726 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c022ba9fae94c04848ba95c378fadd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731469866666 2024-11-13T03:51:11,726 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting de22017ae9954192a82b50006a161a3e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731469866666 2024-11-13T03:51:11,726 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5209e681759e432284c61204c8387a71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:11,726 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b01f029d33041149075fdf71f48e7e2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:11,733 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:11,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T03:51:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,734 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:11,735 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:11,736 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/c665777a2c114bceaacf4b619c6dd536 is 50, key is test_row_0/B:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:11,739 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411131558cfcac7ed4298bc4067f0a0e4b86b_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:11,741 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411131558cfcac7ed4298bc4067f0a0e4b86b_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:11,742 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131558cfcac7ed4298bc4067f0a0e4b86b_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:11,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742036_1212 (size=12697) 2024-11-13T03:51:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c833a291d47c4ce3b513e6dd45610496_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469870047/Put/seqid=0 2024-11-13T03:51:11,756 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/c665777a2c114bceaacf4b619c6dd536 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/c665777a2c114bceaacf4b619c6dd536 2024-11-13T03:51:11,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742037_1213 (size=4469) 2024-11-13T03:51:11,761 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into c665777a2c114bceaacf4b619c6dd536(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:11,761 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:11,761 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=13, startTime=1731469871724; duration=0sec 2024-11-13T03:51:11,761 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:11,761 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:11,761 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:11,763 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:11,763 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#180 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:11,763 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:11,763 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:11,763 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/82dbb29156164b1c89d30d0c22b9de96, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=36.0 K 2024-11-13T03:51:11,763 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 82dbb29156164b1c89d30d0c22b9de96, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731469866342 2024-11-13T03:51:11,764 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/dd9273418165472e8af4351fb2e01d63 is 175, key is test_row_0/A:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:11,764 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bdd5df847c4c76b12a5ba9cce50e31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731469866666 2024-11-13T03:51:11,764 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0582d409604333ab639eb3956c4c3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742038_1214 (size=12454) 2024-11-13T03:51:11,778 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:11,778 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d2a3f346b6e74d1895c55e19a0ad494e is 50, key is test_row_0/C:col10/1731469870038/Put/seqid=0 2024-11-13T03:51:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742039_1215 (size=31651) 2024-11-13T03:51:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742040_1216 (size=12697) 2024-11-13T03:51:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:12,176 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c833a291d47c4ce3b513e6dd45610496_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c833a291d47c4ce3b513e6dd45610496_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/caa5e579c4ad4da4a0fd59681d54a1e2, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/caa5e579c4ad4da4a0fd59681d54a1e2 is 175, key is test_row_0/A:col10/1731469870047/Put/seqid=0 2024-11-13T03:51:12,186 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/dd9273418165472e8af4351fb2e01d63 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63 2024-11-13T03:51:12,193 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/d2a3f346b6e74d1895c55e19a0ad494e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d2a3f346b6e74d1895c55e19a0ad494e 2024-11-13T03:51:12,193 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into dd9273418165472e8af4351fb2e01d63(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:12,193 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:12,193 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=13, startTime=1731469871724; duration=0sec 2024-11-13T03:51:12,193 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:12,193 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742041_1217 (size=31255) 2024-11-13T03:51:12,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/caa5e579c4ad4da4a0fd59681d54a1e2 2024-11-13T03:51:12,200 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into d2a3f346b6e74d1895c55e19a0ad494e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:12,200 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:12,200 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=13, startTime=1731469871724; duration=0sec 2024-11-13T03:51:12,200 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:12,200 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/528dafd271264ec58fcca6e2d8b9752e is 50, key is test_row_0/B:col10/1731469870047/Put/seqid=0 2024-11-13T03:51:12,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742042_1218 (size=12301) 2024-11-13T03:51:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:12,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:12,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469872231, max timestamp=1731469872244Num uniques 859; 2024-11-13T03:51:12,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:51:12,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469872231, max timestamp=1731469872244Num uniques 859; 2024-11-13T03:51:12,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:51:12,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731469872231, max timestamp=1731469872244Num uniques 859; 2024-11-13T03:51:12,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T03:51:12,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469932265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469932265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469932269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469932376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469932376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469932377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469932586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469932586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469932586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/528dafd271264ec58fcca6e2d8b9752e 2024-11-13T03:51:12,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/29d9b48464bb46838a34937188073d3b is 50, key is test_row_0/C:col10/1731469870047/Put/seqid=0 2024-11-13T03:51:12,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742043_1219 (size=12301) 2024-11-13T03:51:12,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469932897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469932897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469932897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,024 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/29d9b48464bb46838a34937188073d3b 2024-11-13T03:51:13,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/caa5e579c4ad4da4a0fd59681d54a1e2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2 2024-11-13T03:51:13,031 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2, entries=150, sequenceid=262, filesize=30.5 K 2024-11-13T03:51:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/528dafd271264ec58fcca6e2d8b9752e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e 2024-11-13T03:51:13,037 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T03:51:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/29d9b48464bb46838a34937188073d3b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b 2024-11-13T03:51:13,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T03:51:13,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1309ms, sequenceid=262, compaction requested=false 2024-11-13T03:51:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T03:51:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T03:51:13,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T03:51:13,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0750 sec 2024-11-13T03:51:13,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0800 sec 2024-11-13T03:51:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T03:51:13,117 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:13,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T03:51:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:13,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:13,120 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:13,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:13,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:13,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113051d6fc8149c4b11989e76a5e943a091_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469872260/Put/seqid=0 2024-11-13T03:51:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742044_1220 (size=12454) 2024-11-13T03:51:13,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:13,288 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113051d6fc8149c4b11989e76a5e943a091_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113051d6fc8149c4b11989e76a5e943a091_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c795e1c82e92428a955c82a81379ab13, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:13,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c795e1c82e92428a955c82a81379ab13 is 175, key is test_row_0/A:col10/1731469872260/Put/seqid=0 2024-11-13T03:51:13,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742045_1221 (size=31255) 2024-11-13T03:51:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:13,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469933421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469933422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469933429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469933536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469933536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469933537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c795e1c82e92428a955c82a81379ab13 2024-11-13T03:51:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/1945099ca9f94ecabcc89d7b0d64db5c is 50, key is test_row_0/B:col10/1731469872260/Put/seqid=0 2024-11-13T03:51:13,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742046_1222 (size=12301) 2024-11-13T03:51:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:13,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469933747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469933747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469933748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469934057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469934057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469934057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,106 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/1945099ca9f94ecabcc89d7b0d64db5c 2024-11-13T03:51:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/a4f4ce788d1b481da3ba4e7445a41319 is 50, key is test_row_0/C:col10/1731469872260/Put/seqid=0 2024-11-13T03:51:14,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742047_1223 (size=12301) 2024-11-13T03:51:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:14,518 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/a4f4ce788d1b481da3ba4e7445a41319 2024-11-13T03:51:14,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/c795e1c82e92428a955c82a81379ab13 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13 2024-11-13T03:51:14,526 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13, entries=150, sequenceid=298, filesize=30.5 K 2024-11-13T03:51:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/1945099ca9f94ecabcc89d7b0d64db5c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c 2024-11-13T03:51:14,533 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T03:51:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/a4f4ce788d1b481da3ba4e7445a41319 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319 2024-11-13T03:51:14,538 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T03:51:14,539 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1267ms, sequenceid=298, compaction requested=true 2024-11-13T03:51:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T03:51:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T03:51:14,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T03:51:14,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-13T03:51:14,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4260 sec 2024-11-13T03:51:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:14,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:51:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:14,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:14,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:14,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:14,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:14,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:14,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111399ffb4540d3d4b0c8d028a93b0c340fd_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742048_1224 (size=14994) 2024-11-13T03:51:14,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:14,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111399ffb4540d3d4b0c8d028a93b0c340fd_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111399ffb4540d3d4b0c8d028a93b0c340fd_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:14,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/d7d018181b784d36936496c669f708a4, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:14,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/d7d018181b784d36936496c669f708a4 is 175, key is test_row_0/A:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742049_1225 (size=39949) 2024-11-13T03:51:14,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/d7d018181b784d36936496c669f708a4 2024-11-13T03:51:14,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/53f3b6d9354b4255ac5976dde9140287 is 50, key is test_row_0/B:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:14,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469934599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469934600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469934601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742050_1226 (size=12301) 2024-11-13T03:51:14,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469934707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469934707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469934717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469934916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469934917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:14,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469934927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:14,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/53f3b6d9354b4255ac5976dde9140287 2024-11-13T03:51:15,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/cabd04182a9d48d7b5592c6a55ab3c9c is 50, key is test_row_0/C:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742051_1227 (size=12301) 2024-11-13T03:51:15,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469935227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469935227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469935236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T03:51:15,267 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:15,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T03:51:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:15,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:15,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:15,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:15,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/cabd04182a9d48d7b5592c6a55ab3c9c 2024-11-13T03:51:15,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T03:51:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:15,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/d7d018181b784d36936496c669f708a4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4 2024-11-13T03:51:15,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4, entries=200, sequenceid=310, filesize=39.0 K 2024-11-13T03:51:15,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/53f3b6d9354b4255ac5976dde9140287 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287 2024-11-13T03:51:15,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T03:51:15,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/cabd04182a9d48d7b5592c6a55ab3c9c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c 2024-11-13T03:51:15,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T03:51:15,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 870ms, sequenceid=310, compaction requested=true 2024-11-13T03:51:15,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:15,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:15,440 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:15,440 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/B is initiating minor compaction (all files) 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134110 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:15,441 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/B in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/A is initiating minor compaction (all files) 2024-11-13T03:51:15,441 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/c665777a2c114bceaacf4b619c6dd536, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=48.4 K 2024-11-13T03:51:15,441 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/A in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,441 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=131.0 K 2024-11-13T03:51:15,441 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4] 2024-11-13T03:51:15,441 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c665777a2c114bceaacf4b619c6dd536, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:15,442 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9273418165472e8af4351fb2e01d63, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:15,442 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 528dafd271264ec58fcca6e2d8b9752e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731469870047 2024-11-13T03:51:15,442 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa5e579c4ad4da4a0fd59681d54a1e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731469870047 2024-11-13T03:51:15,442 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1945099ca9f94ecabcc89d7b0d64db5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731469872257 2024-11-13T03:51:15,442 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c795e1c82e92428a955c82a81379ab13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731469872257 2024-11-13T03:51:15,443 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d018181b784d36936496c669f708a4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731469873419 2024-11-13T03:51:15,443 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f3b6d9354b4255ac5976dde9140287, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731469873419 2024-11-13T03:51:15,449 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:15,452 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:15,453 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/34b1f9c6e49745c7a568a4c1b25a8980 is 50, key is test_row_0/B:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:15,454 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113965b2bcb22a942c8ab3453738a9becb3_5bf9df9b5e7eb4d19bdee05e09462bbe store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:15,457 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113965b2bcb22a942c8ab3453738a9becb3_5bf9df9b5e7eb4d19bdee05e09462bbe, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:15,457 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113965b2bcb22a942c8ab3453738a9becb3_5bf9df9b5e7eb4d19bdee05e09462bbe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742052_1228 (size=12983) 2024-11-13T03:51:15,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742053_1229 (size=4469) 2024-11-13T03:51:15,561 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:51:15,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,574 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c03b4035fb874e4d8d45186bdc9c2d37_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469874592/Put/seqid=0 2024-11-13T03:51:15,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742054_1230 (size=12454) 2024-11-13T03:51:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469935752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469935758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469935758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469935867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469935867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469935867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:15,877 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/34b1f9c6e49745c7a568a4c1b25a8980 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/34b1f9c6e49745c7a568a4c1b25a8980 2024-11-13T03:51:15,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/B of 5bf9df9b5e7eb4d19bdee05e09462bbe into 34b1f9c6e49745c7a568a4c1b25a8980(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:15,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:15,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/B, priority=12, startTime=1731469875440; duration=0sec 2024-11-13T03:51:15,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:15,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:15,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:15,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:15,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 5bf9df9b5e7eb4d19bdee05e09462bbe/C is initiating minor compaction (all files) 2024-11-13T03:51:15,884 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bf9df9b5e7eb4d19bdee05e09462bbe/C in TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:15,884 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d2a3f346b6e74d1895c55e19a0ad494e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp, totalSize=48.4 K 2024-11-13T03:51:15,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a3f346b6e74d1895c55e19a0ad494e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731469868893 2024-11-13T03:51:15,885 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d9b48464bb46838a34937188073d3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731469870047 2024-11-13T03:51:15,885 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f4ce788d1b481da3ba4e7445a41319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731469872257 2024-11-13T03:51:15,885 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting cabd04182a9d48d7b5592c6a55ab3c9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731469873419 2024-11-13T03:51:15,888 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#A#compaction#192 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:15,888 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/af5161b064a845f79ce781c9c9fecb01 is 175, key is test_row_0/A:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:15,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742055_1231 (size=31937) 2024-11-13T03:51:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:15,897 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bf9df9b5e7eb4d19bdee05e09462bbe#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:15,897 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/230d75cfba5643ba814ae2b2fc9b629c is 50, key is test_row_0/C:col10/1731469873419/Put/seqid=0 2024-11-13T03:51:15,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742056_1232 (size=12983) 2024-11-13T03:51:15,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:15,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c03b4035fb874e4d8d45186bdc9c2d37_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c03b4035fb874e4d8d45186bdc9c2d37_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:15,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/cd4f7c5e9944425ca4180a84b3884ce1, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:15,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/cd4f7c5e9944425ca4180a84b3884ce1 is 175, key is test_row_0/A:col10/1731469874592/Put/seqid=0 2024-11-13T03:51:16,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742057_1233 (size=31255) 2024-11-13T03:51:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469936077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469936077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469936077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,297 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/af5161b064a845f79ce781c9c9fecb01 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/af5161b064a845f79ce781c9c9fecb01 2024-11-13T03:51:16,305 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/A of 5bf9df9b5e7eb4d19bdee05e09462bbe into af5161b064a845f79ce781c9c9fecb01(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:16,305 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:16,305 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/A, priority=12, startTime=1731469875439; duration=0sec 2024-11-13T03:51:16,305 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:16,305 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:16,311 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/230d75cfba5643ba814ae2b2fc9b629c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/230d75cfba5643ba814ae2b2fc9b629c 2024-11-13T03:51:16,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bf9df9b5e7eb4d19bdee05e09462bbe/C of 5bf9df9b5e7eb4d19bdee05e09462bbe into 230d75cfba5643ba814ae2b2fc9b629c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:16,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:16,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., storeName=5bf9df9b5e7eb4d19bdee05e09462bbe/C, priority=12, startTime=1731469875440; duration=0sec 2024-11-13T03:51:16,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:16,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469936386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469936386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469936386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,401 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/cd4f7c5e9944425ca4180a84b3884ce1 2024-11-13T03:51:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/a92ebe033cfb44c9bccdb99aea3f513b is 50, key is test_row_0/B:col10/1731469874592/Put/seqid=0 2024-11-13T03:51:16,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742058_1234 (size=12301) 2024-11-13T03:51:16,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/a92ebe033cfb44c9bccdb99aea3f513b 2024-11-13T03:51:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/19504f73d2854118a78e968c17cbb835 is 50, key is test_row_0/C:col10/1731469874592/Put/seqid=0 2024-11-13T03:51:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742059_1235 (size=12301) 2024-11-13T03:51:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469936896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469936907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469936907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:17,223 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/19504f73d2854118a78e968c17cbb835 2024-11-13T03:51:17,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/cd4f7c5e9944425ca4180a84b3884ce1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/cd4f7c5e9944425ca4180a84b3884ce1 2024-11-13T03:51:17,231 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/cd4f7c5e9944425ca4180a84b3884ce1, entries=150, sequenceid=330, filesize=30.5 K 2024-11-13T03:51:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/a92ebe033cfb44c9bccdb99aea3f513b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/a92ebe033cfb44c9bccdb99aea3f513b 2024-11-13T03:51:17,236 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/a92ebe033cfb44c9bccdb99aea3f513b, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T03:51:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/19504f73d2854118a78e968c17cbb835 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/19504f73d2854118a78e968c17cbb835 2024-11-13T03:51:17,241 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/19504f73d2854118a78e968c17cbb835, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T03:51:17,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1668ms, sequenceid=330, compaction requested=false 2024-11-13T03:51:17,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:17,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:17,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T03:51:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T03:51:17,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T03:51:17,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9740 sec 2024-11-13T03:51:17,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9780 sec 2024-11-13T03:51:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T03:51:17,427 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:17,427 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T03:51:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:17,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:17,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:17,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:17,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T03:51:17,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:17,591 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:17,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b9b0117b94ce49c78a561cf67ce42a63_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469875757/Put/seqid=0 2024-11-13T03:51:17,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742060_1236 (size=12454) 2024-11-13T03:51:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. as already flushing 2024-11-13T03:51:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469937939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469937944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:18,011 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b9b0117b94ce49c78a561cf67ce42a63_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9b0117b94ce49c78a561cf67ce42a63_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/4ec414c1344f46f8b2bab2eea0203921, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/4ec414c1344f46f8b2bab2eea0203921 is 175, key is test_row_0/A:col10/1731469875757/Put/seqid=0 2024-11-13T03:51:18,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742061_1237 (size=31255) 2024-11-13T03:51:18,016 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/4ec414c1344f46f8b2bab2eea0203921 2024-11-13T03:51:18,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7934cfd33c8d4298984866986ed15727 is 50, key is test_row_0/B:col10/1731469875757/Put/seqid=0 2024-11-13T03:51:18,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742062_1238 (size=12301) 2024-11-13T03:51:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469938046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469938047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469938047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:18,192 INFO [Thread-732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-732. 2024-11-13T03:51:18,193 DEBUG [Thread-732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,193 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,193 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,195 INFO [Thread-736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-736. 2024-11-13T03:51:18,195 DEBUG [Thread-736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,195 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,195 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,196 INFO [Thread-734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-734. 2024-11-13T03:51:18,196 DEBUG [Thread-734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,196 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,196 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,198 INFO [Thread-738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-738. 2024-11-13T03:51:18,198 DEBUG [Thread-738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,198 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,199 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469938257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469938257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469938258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,435 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7934cfd33c8d4298984866986ed15727 2024-11-13T03:51:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/c24d8f0be47746568a297f47146e4c40 is 50, key is test_row_0/C:col10/1731469875757/Put/seqid=0 2024-11-13T03:51:18,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742063_1239 (size=12301) 2024-11-13T03:51:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50330 deadline: 1731469938568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50344 deadline: 1731469938568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50372 deadline: 1731469938568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:51:18,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/c24d8f0be47746568a297f47146e4c40 2024-11-13T03:51:18,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/4ec414c1344f46f8b2bab2eea0203921 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/4ec414c1344f46f8b2bab2eea0203921 2024-11-13T03:51:18,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/4ec414c1344f46f8b2bab2eea0203921, entries=150, sequenceid=346, filesize=30.5 K 2024-11-13T03:51:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7934cfd33c8d4298984866986ed15727 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7934cfd33c8d4298984866986ed15727 2024-11-13T03:51:18,876 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7934cfd33c8d4298984866986ed15727, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T03:51:18,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/c24d8f0be47746568a297f47146e4c40 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/c24d8f0be47746568a297f47146e4c40 2024-11-13T03:51:18,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/c24d8f0be47746568a297f47146e4c40, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T03:51:18,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1291ms, sequenceid=346, compaction requested=true 2024-11-13T03:51:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T03:51:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T03:51:18,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T03:51:18,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-13T03:51:18,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4580 sec 2024-11-13T03:51:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:18,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:51:18,961 INFO [Thread-725 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-725. 2024-11-13T03:51:18,961 INFO [Thread-729 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-729. 2024-11-13T03:51:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:18,961 DEBUG [Thread-725 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,961 DEBUG [Thread-729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:18,961 DEBUG [Thread-725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,961 DEBUG [Thread-729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:18,961 DEBUG [Thread-725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,961 DEBUG [Thread-729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:18,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:18,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c1ba3475e874676ac6133d5d9d654ee_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_0/A:col10/1731469877942/Put/seqid=0 2024-11-13T03:51:18,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742064_1240 (size=12454) 2024-11-13T03:51:19,083 INFO [Thread-723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-723. 2024-11-13T03:51:19,083 INFO [Thread-727 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-727. 2024-11-13T03:51:19,083 INFO [Thread-721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-721. 2024-11-13T03:51:19,083 DEBUG [Thread-723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:19,083 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 DEBUG [Thread-721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:19,083 DEBUG [Thread-727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:19,083 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:19,083 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:19,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:19,376 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:19,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c1ba3475e874676ac6133d5d9d654ee_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c1ba3475e874676ac6133d5d9d654ee_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:19,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/2471f1550a404dcd8cf63fdc8fc7e84e, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:19,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/2471f1550a404dcd8cf63fdc8fc7e84e is 175, key is test_row_0/A:col10/1731469877942/Put/seqid=0 2024-11-13T03:51:19,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742065_1241 (size=31255) 2024-11-13T03:51:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T03:51:19,579 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3535 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3376 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1420 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4259 rows 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1422 2024-11-13T03:51:19,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4261 rows 2024-11-13T03:51:19,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:51:19,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:51:19,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:19,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:19,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:51:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:19,588 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469879587"}]},"ts":"1731469879587"} 2024-11-13T03:51:19,590 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:51:19,590 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:51:19,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:51:19,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, UNASSIGN}] 2024-11-13T03:51:19,593 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, UNASSIGN 2024-11-13T03:51:19,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:19,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, UNASSIGN because future has completed 2024-11-13T03:51:19,595 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:51:19,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:51:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:19,749 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:19,749 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:51:19,749 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 5bf9df9b5e7eb4d19bdee05e09462bbe, disabling compactions & flushes 2024-11-13T03:51:19,750 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:19,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/2471f1550a404dcd8cf63fdc8fc7e84e 2024-11-13T03:51:19,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7fc98c78e6f64288821b207fc14069bc is 50, key is test_row_0/B:col10/1731469877942/Put/seqid=0 2024-11-13T03:51:19,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742066_1242 (size=12301) 2024-11-13T03:51:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:20,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7fc98c78e6f64288821b207fc14069bc 2024-11-13T03:51:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:20,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/fff6c13c73bc4508957e937b5baf6d4a is 50, key is test_row_0/C:col10/1731469877942/Put/seqid=0 2024-11-13T03:51:20,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742067_1243 (size=12301) 2024-11-13T03:51:20,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/fff6c13c73bc4508957e937b5baf6d4a 2024-11-13T03:51:20,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/2471f1550a404dcd8cf63fdc8fc7e84e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/2471f1550a404dcd8cf63fdc8fc7e84e 2024-11-13T03:51:20,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/2471f1550a404dcd8cf63fdc8fc7e84e, entries=150, sequenceid=367, filesize=30.5 K 2024-11-13T03:51:20,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/7fc98c78e6f64288821b207fc14069bc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7fc98c78e6f64288821b207fc14069bc 2024-11-13T03:51:20,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7fc98c78e6f64288821b207fc14069bc, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T03:51:20,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/fff6c13c73bc4508957e937b5baf6d4a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/fff6c13c73bc4508957e937b5baf6d4a 2024-11-13T03:51:20,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/fff6c13c73bc4508957e937b5baf6d4a, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T03:51:20,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=20.13 KB/20610 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1700ms, sequenceid=367, compaction requested=true 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: 2024-11-13T03:51:20,660 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. after waiting 0 ms 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bf9df9b5e7eb4d19bdee05e09462bbe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. because compaction request was cancelled 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. because compaction request was cancelled 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:A 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:B 2024-11-13T03:51:20,660 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. because compaction request was cancelled 2024-11-13T03:51:20,660 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bf9df9b5e7eb4d19bdee05e09462bbe:C 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=A 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:20,660 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=B 2024-11-13T03:51:20,661 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:20,661 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bf9df9b5e7eb4d19bdee05e09462bbe, store=C 2024-11-13T03:51:20,661 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:20,667 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133dc3698e3f0a48c38dd0e8b83f1b3d25_5bf9df9b5e7eb4d19bdee05e09462bbe is 50, key is test_row_1/A:col10/1731469879079/Put/seqid=0 2024-11-13T03:51:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742068_1244 (size=9914) 2024-11-13T03:51:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:21,073 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:21,080 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133dc3698e3f0a48c38dd0e8b83f1b3d25_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133dc3698e3f0a48c38dd0e8b83f1b3d25_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:21,081 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/f2ff6f43eeff4d3b8ccde288261c5b1a, store: [table=TestAcidGuarantees family=A region=5bf9df9b5e7eb4d19bdee05e09462bbe] 2024-11-13T03:51:21,082 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/f2ff6f43eeff4d3b8ccde288261c5b1a is 175, key is test_row_1/A:col10/1731469879079/Put/seqid=0 2024-11-13T03:51:21,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742069_1245 (size=22561) 2024-11-13T03:51:21,487 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/f2ff6f43eeff4d3b8ccde288261c5b1a 2024-11-13T03:51:21,535 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/89dd10ab45f647ddbc173edebc7cd092 is 50, key is test_row_1/B:col10/1731469879079/Put/seqid=0 2024-11-13T03:51:21,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742070_1246 (size=9857) 2024-11-13T03:51:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:21,942 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/89dd10ab45f647ddbc173edebc7cd092 2024-11-13T03:51:21,956 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/9ca44934721f46bbb7a669c58f71124b is 50, key is test_row_1/C:col10/1731469879079/Put/seqid=0 2024-11-13T03:51:21,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742071_1247 (size=9857) 2024-11-13T03:51:22,361 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/9ca44934721f46bbb7a669c58f71124b 2024-11-13T03:51:22,370 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/A/f2ff6f43eeff4d3b8ccde288261c5b1a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/f2ff6f43eeff4d3b8ccde288261c5b1a 2024-11-13T03:51:22,377 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/f2ff6f43eeff4d3b8ccde288261c5b1a, entries=100, sequenceid=373, filesize=22.0 K 2024-11-13T03:51:22,378 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/B/89dd10ab45f647ddbc173edebc7cd092 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/89dd10ab45f647ddbc173edebc7cd092 2024-11-13T03:51:22,382 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/89dd10ab45f647ddbc173edebc7cd092, entries=100, sequenceid=373, filesize=9.6 K 2024-11-13T03:51:22,383 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/.tmp/C/9ca44934721f46bbb7a669c58f71124b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/9ca44934721f46bbb7a669c58f71124b 2024-11-13T03:51:22,387 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/9ca44934721f46bbb7a669c58f71124b, entries=100, sequenceid=373, filesize=9.6 K 2024-11-13T03:51:22,388 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1728ms, sequenceid=373, compaction requested=true 2024-11-13T03:51:22,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4] to archive 2024-11-13T03:51:22,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:22,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1c194f84e5914b1b8f5c2dc345b7eba0 2024-11-13T03:51:22,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/1df3b5f6ad1a48d59b22d2ad51b968db 2024-11-13T03:51:22,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c16e6e0806f441baafd4d23b90084cf3 2024-11-13T03:51:22,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/176f2713c0144390aff8414f64ae3367 2024-11-13T03:51:22,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c2f32568b4364292ba39b8e62ab420b0 2024-11-13T03:51:22,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/983899c2ff304e828a8ba292196b1750 2024-11-13T03:51:22,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/6507973b5d8b4f24b62095428d2b318b 2024-11-13T03:51:22,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/81bcdb4c695347e8aa3cef4bbedcba2e 2024-11-13T03:51:22,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/53914615f14b40f89c83d4d8e8ce9b4c 2024-11-13T03:51:22,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/80b3045fa9a541c68fc2bfabbc7ff3fe 2024-11-13T03:51:22,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/3479dc60ba88481cb07ab47a8750c93c 2024-11-13T03:51:22,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/55c2fc905e914087897043716a85fe84 2024-11-13T03:51:22,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/ce67a15634b3451baa6622581ecafdcc 2024-11-13T03:51:22,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/b6a58d6dcf65443586016dbad2531cc9 2024-11-13T03:51:22,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/de22017ae9954192a82b50006a161a3e 2024-11-13T03:51:22,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/dd9273418165472e8af4351fb2e01d63 2024-11-13T03:51:22,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/8b01f029d33041149075fdf71f48e7e2 2024-11-13T03:51:22,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/caa5e579c4ad4da4a0fd59681d54a1e2 2024-11-13T03:51:22,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/c795e1c82e92428a955c82a81379ab13 2024-11-13T03:51:22,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/d7d018181b784d36936496c669f708a4 2024-11-13T03:51:22,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/0e4169076dff4b3bbaab87a5bf3de5de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/6f28f305d5b9439d8c2c47999a69e8a9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1196e91c2bd64c2699ce74f2c6b041dd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d9d2c3e59a5546d5a98b5d0f450b5c74, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/c665777a2c114bceaacf4b619c6dd536, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287] to archive 2024-11-13T03:51:22,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:22,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/320076e8ad8c4a96a2e2ed9cc3c8ceb7 2024-11-13T03:51:22,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/b43192a9a094431e92de13d4bbc0f2df 2024-11-13T03:51:22,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/de7bca5cbaf448968711cc8e3d36d948 2024-11-13T03:51:22,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/0e4169076dff4b3bbaab87a5bf3de5de to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/0e4169076dff4b3bbaab87a5bf3de5de 2024-11-13T03:51:22,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/f5d934f603cd4182b485f0b9d66dd66e 2024-11-13T03:51:22,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ca34b601247944ca901288bbc7c4bddd 2024-11-13T03:51:22,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/6f28f305d5b9439d8c2c47999a69e8a9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/6f28f305d5b9439d8c2c47999a69e8a9 2024-11-13T03:51:22,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/e5f3bf81fe3541bbbee9bdcb61276ca1 2024-11-13T03:51:22,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/526182bb61fb4fa2a505c9627292ddec 2024-11-13T03:51:22,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1196e91c2bd64c2699ce74f2c6b041dd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1196e91c2bd64c2699ce74f2c6b041dd 2024-11-13T03:51:22,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5bd78907b3f146b0ba5d83d9d1db991a 2024-11-13T03:51:22,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/ec4c1c5e2ab9403889b00cc4756d9f74 2024-11-13T03:51:22,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d9d2c3e59a5546d5a98b5d0f450b5c74 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d9d2c3e59a5546d5a98b5d0f450b5c74 2024-11-13T03:51:22,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/d4b83b28e1874ff8ac134c178fd269de 2024-11-13T03:51:22,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/2c022ba9fae94c04848ba95c378fadd4 2024-11-13T03:51:22,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/c665777a2c114bceaacf4b619c6dd536 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/c665777a2c114bceaacf4b619c6dd536 2024-11-13T03:51:22,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/5209e681759e432284c61204c8387a71 2024-11-13T03:51:22,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/528dafd271264ec58fcca6e2d8b9752e 2024-11-13T03:51:22,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/1945099ca9f94ecabcc89d7b0d64db5c 2024-11-13T03:51:22,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/53f3b6d9354b4255ac5976dde9140287 2024-11-13T03:51:22,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/04ce30d3724347b78f0b3dddc7b0c8fc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d1fb15bebf0a41449aa9481271ca6b81, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/054c7fccbdd1455788de41a80f3af29e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/82dbb29156164b1c89d30d0c22b9de96, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d2a3f346b6e74d1895c55e19a0ad494e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c] to archive 2024-11-13T03:51:22,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:22,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/bb49611b708d4babba207c40009953f8 2024-11-13T03:51:22,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/5b515ca09ac4467d8fc4972c32f2b136 2024-11-13T03:51:22,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/e9b6bf14954c40f98f6aa4dcbec5a068 2024-11-13T03:51:22,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/04ce30d3724347b78f0b3dddc7b0c8fc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/04ce30d3724347b78f0b3dddc7b0c8fc 2024-11-13T03:51:22,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4c4c34764f094a6dbcec3323af02cc53 2024-11-13T03:51:22,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/58b9b70985db4ae386f509bfd8663019 2024-11-13T03:51:22,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d1fb15bebf0a41449aa9481271ca6b81 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d1fb15bebf0a41449aa9481271ca6b81 2024-11-13T03:51:22,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d4c02986756043748f5302b201a160c9 2024-11-13T03:51:22,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/2dab0acddfe44a52bfccb6ff88a5fd88 2024-11-13T03:51:22,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/054c7fccbdd1455788de41a80f3af29e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/054c7fccbdd1455788de41a80f3af29e 2024-11-13T03:51:22,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/577699881ae843d98f1bf7efa0deee86 2024-11-13T03:51:22,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/de6560bfeb3e415f98a400361d3ce426 2024-11-13T03:51:22,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/82dbb29156164b1c89d30d0c22b9de96 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/82dbb29156164b1c89d30d0c22b9de96 2024-11-13T03:51:22,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/4bb38c6815cf4dc1b5968e722d125855 2024-11-13T03:51:22,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/b4bdd5df847c4c76b12a5ba9cce50e31 2024-11-13T03:51:22,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d2a3f346b6e74d1895c55e19a0ad494e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/d2a3f346b6e74d1895c55e19a0ad494e 2024-11-13T03:51:22,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/1d0582d409604333ab639eb3956c4c3d 2024-11-13T03:51:22,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/29d9b48464bb46838a34937188073d3b 2024-11-13T03:51:22,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/a4f4ce788d1b481da3ba4e7445a41319 2024-11-13T03:51:22,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/cabd04182a9d48d7b5592c6a55ab3c9c 2024-11-13T03:51:22,456 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=4 2024-11-13T03:51:22,457 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. 2024-11-13T03:51:22,457 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 5bf9df9b5e7eb4d19bdee05e09462bbe: Waiting for close lock at 1731469879749Running coprocessor pre-close hooks at 1731469879749Disabling compacts and flushes for region at 1731469879749Disabling writes for close at 1731469880660 (+911 ms)Obtaining lock to block concurrent updates at 1731469880660Preparing flush snapshotting stores in 5bf9df9b5e7eb4d19bdee05e09462bbe at 1731469880660Finished memstore snapshotting TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731469880661 (+1 ms)Flushing stores of TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe. at 1731469880661Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/A: creating writer at 1731469880661Flushing mob file 5bf9df9b5e7eb4d19bdee05e09462bbe/A: appending metadata at 1731469880666 (+5 ms)Flushing mob file 5bf9df9b5e7eb4d19bdee05e09462bbe/A: closing flushed file at 1731469880666Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/A: appending metadata at 1731469881081 (+415 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/A: closing flushed file at 1731469881082 (+1 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/B: creating writer at 1731469881531 (+449 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/B: appending metadata at 1731469881534 (+3 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/B: closing flushed file at 1731469881534Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/C: creating writer at 1731469881953 (+419 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/C: appending metadata at 1731469881956 (+3 ms)Flushing 5bf9df9b5e7eb4d19bdee05e09462bbe/C: closing flushed file at 1731469881956Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5234c941: reopening flushed file at 1731469882369 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4715aa6d: reopening flushed file at 1731469882377 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11dd09d8: reopening flushed file at 1731469882383 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5bf9df9b5e7eb4d19bdee05e09462bbe in 1728ms, sequenceid=373, compaction requested=true at 1731469882388 (+5 ms)Writing region close event to WAL at 1731469882453 (+65 ms)Running coprocessor post-close hooks at 1731469882457 (+4 ms)Closed at 1731469882457 2024-11-13T03:51:22,458 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:22,459 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5bf9df9b5e7eb4d19bdee05e09462bbe, regionState=CLOSED 2024-11-13T03:51:22,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:51:22,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T03:51:22,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bf9df9b5e7eb4d19bdee05e09462bbe, server=ac1a51731247,42123,1731469817167 in 2.8660 sec 2024-11-13T03:51:22,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-13T03:51:22,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bf9df9b5e7eb4d19bdee05e09462bbe, UNASSIGN in 2.8700 sec 2024-11-13T03:51:22,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T03:51:22,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8740 sec 2024-11-13T03:51:22,466 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469882466"}]},"ts":"1731469882466"} 2024-11-13T03:51:22,467 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:51:22,467 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:51:22,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8860 sec 2024-11-13T03:51:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T03:51:23,759 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:23,760 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:51:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T03:51:23,765 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,769 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,773 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits] 2024-11-13T03:51:23,778 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/2471f1550a404dcd8cf63fdc8fc7e84e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/2471f1550a404dcd8cf63fdc8fc7e84e 2024-11-13T03:51:23,779 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/4ec414c1344f46f8b2bab2eea0203921 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/4ec414c1344f46f8b2bab2eea0203921 2024-11-13T03:51:23,780 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/af5161b064a845f79ce781c9c9fecb01 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/af5161b064a845f79ce781c9c9fecb01 2024-11-13T03:51:23,781 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/cd4f7c5e9944425ca4180a84b3884ce1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/cd4f7c5e9944425ca4180a84b3884ce1 2024-11-13T03:51:23,782 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/f2ff6f43eeff4d3b8ccde288261c5b1a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/A/f2ff6f43eeff4d3b8ccde288261c5b1a 2024-11-13T03:51:23,784 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/34b1f9c6e49745c7a568a4c1b25a8980 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/34b1f9c6e49745c7a568a4c1b25a8980 2024-11-13T03:51:23,785 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7934cfd33c8d4298984866986ed15727 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7934cfd33c8d4298984866986ed15727 2024-11-13T03:51:23,786 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7fc98c78e6f64288821b207fc14069bc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/7fc98c78e6f64288821b207fc14069bc 2024-11-13T03:51:23,787 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/89dd10ab45f647ddbc173edebc7cd092 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/89dd10ab45f647ddbc173edebc7cd092 2024-11-13T03:51:23,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/a92ebe033cfb44c9bccdb99aea3f513b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/B/a92ebe033cfb44c9bccdb99aea3f513b 2024-11-13T03:51:23,790 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/19504f73d2854118a78e968c17cbb835 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/19504f73d2854118a78e968c17cbb835 2024-11-13T03:51:23,791 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/230d75cfba5643ba814ae2b2fc9b629c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/230d75cfba5643ba814ae2b2fc9b629c 2024-11-13T03:51:23,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/9ca44934721f46bbb7a669c58f71124b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/9ca44934721f46bbb7a669c58f71124b 2024-11-13T03:51:23,793 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/c24d8f0be47746568a297f47146e4c40 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/c24d8f0be47746568a297f47146e4c40 2024-11-13T03:51:23,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/fff6c13c73bc4508957e937b5baf6d4a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/C/fff6c13c73bc4508957e937b5baf6d4a 2024-11-13T03:51:23,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits/376.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe/recovered.edits/376.seqid 2024-11-13T03:51:23,799 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,799 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:51:23,799 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:51:23,800 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T03:51:23,804 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113051d6fc8149c4b11989e76a5e943a091_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113051d6fc8149c4b11989e76a5e943a091_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111307b7a8744dfe4537a7212bc50c96c701_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111307b7a8744dfe4537a7212bc50c96c701_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310baed3a89a24beb864e78a72e4b33b3_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310baed3a89a24beb864e78a72e4b33b3_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,807 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a83c5e6f608475dabb7a8d35e3ca905_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133a83c5e6f608475dabb7a8d35e3ca905_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,808 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133dc3698e3f0a48c38dd0e8b83f1b3d25_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133dc3698e3f0a48c38dd0e8b83f1b3d25_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,809 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361d29f614bdf4e768b24050961758eaa_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361d29f614bdf4e768b24050961758eaa_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,810 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f47f4a8aac74f879420ac617acc27b5_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f47f4a8aac74f879420ac617acc27b5_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,811 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f52164c189344d78c9c0dec5698ce51_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137f52164c189344d78c9c0dec5698ce51_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,812 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389f9f58d0671469eaec7631ed2a5fdc3_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389f9f58d0671469eaec7631ed2a5fdc3_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,812 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138d812ccf0ae545bdaede39bbf79f386a_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138d812ccf0ae545bdaede39bbf79f386a_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,813 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111399ffb4540d3d4b0c8d028a93b0c340fd_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111399ffb4540d3d4b0c8d028a93b0c340fd_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,814 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c1ba3475e874676ac6133d5d9d654ee_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c1ba3475e874676ac6133d5d9d654ee_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,815 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3f51c31614c48e6adc7c88b187f050c_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b3f51c31614c48e6adc7c88b187f050c_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,816 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9b0117b94ce49c78a561cf67ce42a63_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9b0117b94ce49c78a561cf67ce42a63_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,817 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c03b4035fb874e4d8d45186bdc9c2d37_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c03b4035fb874e4d8d45186bdc9c2d37_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,818 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c833a291d47c4ce3b513e6dd45610496_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c833a291d47c4ce3b513e6dd45610496_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,819 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9de1705f50f487a962375c4a868fa85_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9de1705f50f487a962375c4a868fa85_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,820 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed877a62903046de9032e321a3ee8538_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed877a62903046de9032e321a3ee8538_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,820 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f14e75fc0aa44a4b9f47ad361dc65765_5bf9df9b5e7eb4d19bdee05e09462bbe to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f14e75fc0aa44a4b9f47ad361dc65765_5bf9df9b5e7eb4d19bdee05e09462bbe 2024-11-13T03:51:23,821 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:51:23,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,825 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:51:23,827 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:51:23,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:51:23,828 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731469883828"}]},"ts":"9223372036854775807"} 2024-11-13T03:51:23,831 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:51:23,831 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5bf9df9b5e7eb4d19bdee05e09462bbe, NAME => 'TestAcidGuarantees,,1731469855018.5bf9df9b5e7eb4d19bdee05e09462bbe.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:51:23,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:51:23,831 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731469883831"}]},"ts":"9223372036854775807"} 2024-11-13T03:51:23,833 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:51:23,834 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-13T03:51:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T03:51:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:51:23,867 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:23,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-36274116_22 at /127.0.0.1:41244 [Waiting for operation #163] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_431975313_22 at /127.0.0.1:41254 [Waiting for operation #160] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-36274116_22 at /127.0.0.1:41162 [Waiting for operation #192] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 261) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8260 (was 8364) 2024-11-13T03:51:23,883 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=8260 2024-11-13T03:51:23,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:51:23,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:51:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:23,887 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:51:23,887 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:23,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-13T03:51:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:23,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:51:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742072_1248 (size=1061) 2024-11-13T03:51:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:24,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:51:24,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742073_1249 (size=53) 2024-11-13T03:51:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:24,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:51:24,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e3e30c23eeb6b0b0afd7668c5b9a55d3, disabling compactions & flushes 2024-11-13T03:51:24,712 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:24,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:24,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. after waiting 0 ms 2024-11-13T03:51:24,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:24,712 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:24,713 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: Waiting for close lock at 1731469884712Disabling compacts and flushes for region at 1731469884712Disabling writes for close at 1731469884712Writing region close event to WAL at 1731469884712Closed at 1731469884712 2024-11-13T03:51:24,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:51:24,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469884715"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469884715"}]},"ts":"1731469884715"} 2024-11-13T03:51:24,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:51:24,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:51:24,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469884721"}]},"ts":"1731469884721"} 2024-11-13T03:51:24,723 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:51:24,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, ASSIGN}] 2024-11-13T03:51:24,724 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, ASSIGN 2024-11-13T03:51:24,725 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:51:24,876 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e3e30c23eeb6b0b0afd7668c5b9a55d3, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:24,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, ASSIGN because future has completed 2024-11-13T03:51:24,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:51:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:25,035 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:25,035 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:51:25,035 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,035 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:51:25,035 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,035 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,037 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,038 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:25,038 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3e30c23eeb6b0b0afd7668c5b9a55d3 columnFamilyName A 2024-11-13T03:51:25,038 DEBUG [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:25,038 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(327): Store=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:25,038 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,039 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:25,040 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3e30c23eeb6b0b0afd7668c5b9a55d3 columnFamilyName B 2024-11-13T03:51:25,040 DEBUG [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:25,040 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(327): Store=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:25,040 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,041 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:25,041 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3e30c23eeb6b0b0afd7668c5b9a55d3 columnFamilyName C 2024-11-13T03:51:25,041 DEBUG [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:25,041 INFO [StoreOpener-e3e30c23eeb6b0b0afd7668c5b9a55d3-1 {}] regionserver.HStore(327): Store=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:25,042 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:25,042 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,042 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,043 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,043 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,043 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,044 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:51:25,045 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,047 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:51:25,047 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened e3e30c23eeb6b0b0afd7668c5b9a55d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59559638, jitterRate=-0.11249223351478577}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:51:25,047 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:25,048 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: Running coprocessor pre-open hook at 1731469885035Writing region info on filesystem at 1731469885035Initializing all the Stores at 1731469885036 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469885036Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469885036Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469885036Cleaning up temporary data from old regions at 1731469885043 (+7 ms)Running coprocessor post-open hooks at 1731469885047 (+4 ms)Region opened successfully at 1731469885048 (+1 ms) 2024-11-13T03:51:25,049 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., pid=65, masterSystemTime=1731469885031 2024-11-13T03:51:25,050 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:25,050 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:25,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e3e30c23eeb6b0b0afd7668c5b9a55d3, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:25,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:51:25,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T03:51:25,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 in 175 msec 2024-11-13T03:51:25,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-13T03:51:25,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, ASSIGN in 331 msec 2024-11-13T03:51:25,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:51:25,058 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469885057"}]},"ts":"1731469885057"} 2024-11-13T03:51:25,059 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:51:25,060 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:51:25,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-13T03:51:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-13T03:51:26,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:26,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a01c226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,047 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154ab8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,048 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,051 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:51:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:51:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7976b547 2024-11-13T03:51:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:51:26,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:51:26,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,056 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f272c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15154824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,061 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6dcc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,066 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e88f496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,067 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22107d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,069 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7f0938, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c6788b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,072 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da676e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,074 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad9124d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,075 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66dc73b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70df6780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,078 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25840113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,079 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134c62cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,081 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42283f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,082 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bcdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,084 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61aa3e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,085 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb79441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:26,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:26,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:26,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2c8db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:26,089 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:26,089 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,089 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,089 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,090 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T03:51:26,091 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:26,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,093 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:26,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,100 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,100 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:51:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:26,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,109 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,111 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:26,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:26,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469886122, max timestamp=1731469886137Num uniques 859; 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469886122, max timestamp=1731469886137Num uniques 859; 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469886122, max timestamp=1731469886137Num uniques 859; 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:26,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:26,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/bb4dd0f0f8f04656b14fd7ea4714e36e is 50, key is test_row_0/A:col10/1731469886099/Put/seqid=0 2024-11-13T03:51:26,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469946161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469946161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469946162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469886158, max timestamp=1731469886165Num uniques 1018; 2024-11-13T03:51:26,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:51:26,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469886158, max timestamp=1731469886165Num uniques 1018; 2024-11-13T03:51:26,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:51:26,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742074_1250 (size=9657) 2024-11-13T03:51:26,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731469886158, max timestamp=1731469886165Num uniques 1018; 2024-11-13T03:51:26,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T03:51:26,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/bb4dd0f0f8f04656b14fd7ea4714e36e 2024-11-13T03:51:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:26,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bf881d00f5ac4ec08977bae965d9f128 is 50, key is test_row_0/B:col10/1731469886099/Put/seqid=0 2024-11-13T03:51:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469946227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742075_1251 (size=9657) 2024-11-13T03:51:26,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bf881d00f5ac4ec08977bae965d9f128 2024-11-13T03:51:26,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469946229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T03:51:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 is 50, key is test_row_0/C:col10/1731469886099/Put/seqid=0 2024-11-13T03:51:26,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742076_1252 (size=9657) 2024-11-13T03:51:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469946277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469946277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469946278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469946337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469946347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T03:51:26,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:26,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469946486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469946487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469946487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469946546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T03:51:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:26,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469946556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 2024-11-13T03:51:26,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/bb4dd0f0f8f04656b14fd7ea4714e36e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e 2024-11-13T03:51:26,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e, entries=100, sequenceid=11, filesize=9.4 K 2024-11-13T03:51:26,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bf881d00f5ac4ec08977bae965d9f128 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128 2024-11-13T03:51:26,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128, entries=100, sequenceid=11, filesize=9.4 K 2024-11-13T03:51:26,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 2024-11-13T03:51:26,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7, entries=100, sequenceid=11, filesize=9.4 K 2024-11-13T03:51:26,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 592ms, sequenceid=11, compaction requested=false 2024-11-13T03:51:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:26,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:26,708 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/90b01ee7dff74326a356889438bad06b is 50, key is test_row_0/A:col10/1731469886225/Put/seqid=0 2024-11-13T03:51:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:26,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742077_1253 (size=12001) 2024-11-13T03:51:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:26,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469946806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469946807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469946808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469946857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469946866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469946917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469946917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469946917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/90b01ee7dff74326a356889438bad06b 2024-11-13T03:51:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/c6df0e874bf64467b6388c2eaebe1deb is 50, key is test_row_0/B:col10/1731469886225/Put/seqid=0 2024-11-13T03:51:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469947127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469947127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469947127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742078_1254 (size=12001) 2024-11-13T03:51:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469947367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469947377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469947436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469947437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469947437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/c6df0e874bf64467b6388c2eaebe1deb 2024-11-13T03:51:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/610b622aa17942beaae6d771c20a4f81 is 50, key is test_row_0/C:col10/1731469886225/Put/seqid=0 2024-11-13T03:51:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742079_1255 (size=12001) 2024-11-13T03:51:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469947947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/610b622aa17942beaae6d771c20a4f81 2024-11-13T03:51:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469947947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469947947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/90b01ee7dff74326a356889438bad06b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b 2024-11-13T03:51:27,970 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T03:51:27,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/c6df0e874bf64467b6388c2eaebe1deb as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb 2024-11-13T03:51:27,976 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T03:51:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/610b622aa17942beaae6d771c20a4f81 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81 2024-11-13T03:51:27,978 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:51:27,982 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T03:51:27,983 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1275ms, sequenceid=64, compaction requested=false 2024-11-13T03:51:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T03:51:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T03:51:27,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T03:51:27,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-13T03:51:27,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8990 sec 2024-11-13T03:51:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T03:51:28,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:28,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T03:51:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:28,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:28,240 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:28,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:28,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:28,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:28,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/17a6af06db5f4d91bf837bed9894e854 is 50, key is test_row_0/A:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:28,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742080_1256 (size=14341) 2024-11-13T03:51:28,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/17a6af06db5f4d91bf837bed9894e854 2024-11-13T03:51:28,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bd8cc59fca45459fa05b240b97b0b6c2 is 50, key is test_row_0/B:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:28,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742081_1257 (size=12001) 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469888415, max timestamp=1731469888431Num uniques 859; 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469888415, max timestamp=1731469888431Num uniques 859; 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469888415, max timestamp=1731469888431Num uniques 859; 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:28,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:28,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469948464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469948465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:28,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469948577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469948577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469948786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469948787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bd8cc59fca45459fa05b240b97b0b6c2 2024-11-13T03:51:28,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8f6c89a45ca342f79cf2bd0d40c6326c is 50, key is test_row_0/C:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:28,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742082_1258 (size=12001) 2024-11-13T03:51:28,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:28,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:28,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:28,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469948967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,970 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469948976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469948976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:28,980 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:28,980 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469949096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469949097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:29,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8f6c89a45ca342f79cf2bd0d40c6326c 2024-11-13T03:51:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/17a6af06db5f4d91bf837bed9894e854 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854 2024-11-13T03:51:29,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854, entries=200, sequenceid=75, filesize=14.0 K 2024-11-13T03:51:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bd8cc59fca45459fa05b240b97b0b6c2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2 2024-11-13T03:51:29,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T03:51:29,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8f6c89a45ca342f79cf2bd0d40c6326c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c 2024-11-13T03:51:29,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T03:51:29,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 861ms, sequenceid=75, compaction requested=true 2024-11-13T03:51:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:29,242 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:29,242 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:29,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:29,242 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:29,243 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,243 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=32.9 K 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:29,243 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf881d00f5ac4ec08977bae965d9f128, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469886099 2024-11-13T03:51:29,243 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=35.2 K 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6df0e874bf64467b6388c2eaebe1deb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731469886215 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4dd0f0f8f04656b14fd7ea4714e36e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469886099 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8cc59fca45459fa05b240b97b0b6c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:29,243 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b01ee7dff74326a356889438bad06b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731469886215 2024-11-13T03:51:29,244 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a6af06db5f4d91bf837bed9894e854, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:29,250 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:29,250 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:29,251 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d63afa8475c344adb6045e49a6bb312d is 50, key is test_row_0/A:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:29,251 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/66b244112eda41748b9fddd33f0aff44 is 50, key is test_row_0/B:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:29,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742083_1259 (size=12104) 2024-11-13T03:51:29,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742084_1260 (size=12104) 2024-11-13T03:51:29,279 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d63afa8475c344adb6045e49a6bb312d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d63afa8475c344adb6045e49a6bb312d 2024-11-13T03:51:29,286 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into d63afa8475c344adb6045e49a6bb312d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:29,286 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:29,286 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469889241; duration=0sec 2024-11-13T03:51:29,286 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:29,286 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:29,286 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:29,287 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:29,287 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:29,287 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,287 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/66b244112eda41748b9fddd33f0aff44 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/66b244112eda41748b9fddd33f0aff44 2024-11-13T03:51:29,287 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=32.9 K 2024-11-13T03:51:29,288 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d24b9233e8df4d5bbc2dcdb1aa5853a7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469886099 2024-11-13T03:51:29,288 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 610b622aa17942beaae6d771c20a4f81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731469886215 2024-11-13T03:51:29,288 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f6c89a45ca342f79cf2bd0d40c6326c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:29,292 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 66b244112eda41748b9fddd33f0aff44(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:29,292 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:29,292 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469889242; duration=0sec 2024-11-13T03:51:29,292 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:29,292 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:29,296 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:29,296 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/93ead78c443d40bf911c444c063f80ac is 50, key is test_row_0/C:col10/1731469886805/Put/seqid=0 2024-11-13T03:51:29,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742085_1261 (size=12104) 2024-11-13T03:51:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T03:51:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:29,308 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T03:51:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/95dfedde23cb4632ab2c7751d28ef3c3 is 50, key is test_row_0/A:col10/1731469888463/Put/seqid=0 2024-11-13T03:51:29,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742086_1262 (size=12001) 2024-11-13T03:51:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:29,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:29,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469949623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469949624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,709 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/93ead78c443d40bf911c444c063f80ac as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/93ead78c443d40bf911c444c063f80ac 2024-11-13T03:51:29,713 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 93ead78c443d40bf911c444c063f80ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:29,713 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:29,714 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469889242; duration=0sec 2024-11-13T03:51:29,714 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:29,714 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:29,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/95dfedde23cb4632ab2c7751d28ef3c3 2024-11-13T03:51:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/788c28bf307a49ffa09c32f835d113f1 is 50, key is test_row_0/B:col10/1731469888463/Put/seqid=0 2024-11-13T03:51:29,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742087_1263 (size=12001) 2024-11-13T03:51:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469949737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469949737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469949947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469949947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,131 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/788c28bf307a49ffa09c32f835d113f1 2024-11-13T03:51:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/f8ce7230b0d14829b5413448d55ef01f is 50, key is test_row_0/C:col10/1731469888463/Put/seqid=0 2024-11-13T03:51:30,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742088_1264 (size=12001) 2024-11-13T03:51:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469950257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469950257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:30,541 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/f8ce7230b0d14829b5413448d55ef01f 2024-11-13T03:51:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/95dfedde23cb4632ab2c7751d28ef3c3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3 2024-11-13T03:51:30,549 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T03:51:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/788c28bf307a49ffa09c32f835d113f1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1 2024-11-13T03:51:30,554 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T03:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/f8ce7230b0d14829b5413448d55ef01f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f 2024-11-13T03:51:30,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T03:51:30,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1251ms, sequenceid=112, compaction requested=false 2024-11-13T03:51:30,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:30,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T03:51:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T03:51:30,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T03:51:30,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-13T03:51:30,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3260 sec 2024-11-13T03:51:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:30,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d981d334406542c5a088ca2e0c519a3e is 50, key is test_row_0/A:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:30,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742089_1265 (size=12001) 2024-11-13T03:51:30,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469950805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469950806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469950917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469950917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469950996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469950996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,000 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:31,001 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469951007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,008 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469951127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469951127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d981d334406542c5a088ca2e0c519a3e 2024-11-13T03:51:31,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/8c141c9d90e14676bbed8a65b3d16592 is 50, key is test_row_0/B:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:31,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742090_1266 (size=12001) 2024-11-13T03:51:31,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/8c141c9d90e14676bbed8a65b3d16592 2024-11-13T03:51:31,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/51a4040e823748a4903da41e576f65c3 is 50, key is test_row_0/C:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:31,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742091_1267 (size=12001) 2024-11-13T03:51:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469951437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469951437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/51a4040e823748a4903da41e576f65c3 2024-11-13T03:51:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d981d334406542c5a088ca2e0c519a3e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e 2024-11-13T03:51:31,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e, entries=150, sequenceid=125, filesize=11.7 K 2024-11-13T03:51:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/8c141c9d90e14676bbed8a65b3d16592 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592 2024-11-13T03:51:31,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592, entries=150, sequenceid=125, filesize=11.7 K 2024-11-13T03:51:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/51a4040e823748a4903da41e576f65c3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3 2024-11-13T03:51:31,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3, entries=150, sequenceid=125, filesize=11.7 K 2024-11-13T03:51:31,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 878ms, sequenceid=125, compaction requested=true 2024-11-13T03:51:31,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:31,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:31,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:31,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:31,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:31,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:31,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:31,649 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:31,650 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:31,650 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:31,650 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/66b244112eda41748b9fddd33f0aff44, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=35.3 K 2024-11-13T03:51:31,650 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d63afa8475c344adb6045e49a6bb312d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=35.3 K 2024-11-13T03:51:31,650 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b244112eda41748b9fddd33f0aff44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:31,650 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d63afa8475c344adb6045e49a6bb312d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:31,650 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 788c28bf307a49ffa09c32f835d113f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731469888458 2024-11-13T03:51:31,650 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dfedde23cb4632ab2c7751d28ef3c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731469888458 2024-11-13T03:51:31,651 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c141c9d90e14676bbed8a65b3d16592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:31,651 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d981d334406542c5a088ca2e0c519a3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:31,670 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:31,670 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0d15c7ff65d9487e90da336756355e59 is 50, key is test_row_0/B:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742092_1268 (size=12207) 2024-11-13T03:51:31,681 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:31,682 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/4caff88d38934aadaabfcd49646760a8 is 50, key is test_row_0/A:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:31,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742093_1269 (size=12207) 2024-11-13T03:51:31,688 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0d15c7ff65d9487e90da336756355e59 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0d15c7ff65d9487e90da336756355e59 2024-11-13T03:51:31,693 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 0d15c7ff65d9487e90da336756355e59(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:31,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:31,693 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469891648; duration=0sec 2024-11-13T03:51:31,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:31,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:31,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:31,694 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:31,694 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:31,694 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:31,694 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/93ead78c443d40bf911c444c063f80ac, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=35.3 K 2024-11-13T03:51:31,695 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ead78c443d40bf911c444c063f80ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731469886805 2024-11-13T03:51:31,695 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ce7230b0d14829b5413448d55ef01f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731469888458 2024-11-13T03:51:31,695 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a4040e823748a4903da41e576f65c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:31,703 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:31,703 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/203d4e2e78544d50946dfce60d456795 is 50, key is test_row_0/C:col10/1731469890769/Put/seqid=0 2024-11-13T03:51:31,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742094_1270 (size=12207) 2024-11-13T03:51:31,720 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/203d4e2e78544d50946dfce60d456795 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/203d4e2e78544d50946dfce60d456795 2024-11-13T03:51:31,729 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 203d4e2e78544d50946dfce60d456795(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:31,729 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:31,729 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469891649; duration=0sec 2024-11-13T03:51:31,729 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:31,729 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:31,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:31,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:31,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/5339ea3f880d480989f4e1c7da9067c6 is 50, key is test_row_0/A:col10/1731469890800/Put/seqid=0 2024-11-13T03:51:31,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742095_1271 (size=14541) 2024-11-13T03:51:31,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469951966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:31,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469951967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469952076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469952076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/4caff88d38934aadaabfcd49646760a8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4caff88d38934aadaabfcd49646760a8 2024-11-13T03:51:32,094 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 4caff88d38934aadaabfcd49646760a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:32,095 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:32,095 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469891648; duration=0sec 2024-11-13T03:51:32,095 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:32,095 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469952287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469952287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/5339ea3f880d480989f4e1c7da9067c6 2024-11-13T03:51:32,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bdb9c9c562ac4dbe817cf2d148f4d1cd is 50, key is test_row_0/B:col10/1731469890800/Put/seqid=0 2024-11-13T03:51:32,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742096_1272 (size=12151) 2024-11-13T03:51:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T03:51:32,397 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:32,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T03:51:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:32,399 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:32,400 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:32,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:32,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469952597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469952597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:32,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:32,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bdb9c9c562ac4dbe817cf2d148f4d1cd 2024-11-13T03:51:32,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6d3fb5e7b5e649d1ba2f73aee6180660 is 50, key is test_row_0/C:col10/1731469890800/Put/seqid=0 2024-11-13T03:51:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742097_1273 (size=12151) 2024-11-13T03:51:32,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:32,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469953107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469953107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:33,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:33,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6d3fb5e7b5e649d1ba2f73aee6180660 2024-11-13T03:51:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/5339ea3f880d480989f4e1c7da9067c6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6 2024-11-13T03:51:33,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6, entries=200, sequenceid=149, filesize=14.2 K 2024-11-13T03:51:33,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/bdb9c9c562ac4dbe817cf2d148f4d1cd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd 2024-11-13T03:51:33,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T03:51:33,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6d3fb5e7b5e649d1ba2f73aee6180660 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660 2024-11-13T03:51:33,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T03:51:33,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1254ms, sequenceid=149, compaction requested=false 2024-11-13T03:51:33,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:33,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T03:51:33,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,316 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 is 50, key is test_row_0/A:col10/1731469891965/Put/seqid=0 2024-11-13T03:51:33,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742098_1274 (size=9757) 2024-11-13T03:51:33,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 2024-11-13T03:51:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/e9a8bbbcc2254199bb03a9b5f1bbb7df is 50, key is test_row_0/B:col10/1731469891965/Put/seqid=0 2024-11-13T03:51:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742099_1275 (size=9757) 2024-11-13T03:51:33,369 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/e9a8bbbcc2254199bb03a9b5f1bbb7df 2024-11-13T03:51:33,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/08e63a151053463cb5ee0f5943e49d34 is 50, key is test_row_0/C:col10/1731469891965/Put/seqid=0 2024-11-13T03:51:33,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742100_1276 (size=9757) 2024-11-13T03:51:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:33,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/08e63a151053463cb5ee0f5943e49d34 2024-11-13T03:51:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 2024-11-13T03:51:33,794 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53, entries=100, sequenceid=160, filesize=9.5 K 2024-11-13T03:51:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/e9a8bbbcc2254199bb03a9b5f1bbb7df as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df 2024-11-13T03:51:33,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df, entries=100, sequenceid=160, filesize=9.5 K 2024-11-13T03:51:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/08e63a151053463cb5ee0f5943e49d34 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34 2024-11-13T03:51:33,804 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34, entries=100, sequenceid=160, filesize=9.5 K 2024-11-13T03:51:33,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 489ms, sequenceid=160, compaction requested=true 2024-11-13T03:51:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T03:51:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T03:51:33,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T03:51:33,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4060 sec 2024-11-13T03:51:33,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4100 sec 2024-11-13T03:51:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:34,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:34,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:34,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/e394091067fd4a58950e7f838c1703eb is 50, key is test_row_0/A:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742101_1277 (size=16931) 2024-11-13T03:51:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469954166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469894159, max timestamp=1731469894169Num uniques 859; 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469894159, max timestamp=1731469894169Num uniques 859; 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469894159, max timestamp=1731469894169Num uniques 859; 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:34,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469954248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469954277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469954356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469954487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/e394091067fd4a58950e7f838c1703eb 2024-11-13T03:51:34,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4a84a56d6f7a4e86864c0a8718235c79 is 50, key is test_row_0/B:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742102_1278 (size=12151) 2024-11-13T03:51:34,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4a84a56d6f7a4e86864c0a8718235c79 2024-11-13T03:51:34,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9342ba649e814b6e804205218009ee87 is 50, key is test_row_0/C:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T03:51:34,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:34,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T03:51:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T03:51:34,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:34,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:34,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742103_1279 (size=12151) 2024-11-13T03:51:34,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9342ba649e814b6e804205218009ee87 2024-11-13T03:51:34,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/e394091067fd4a58950e7f838c1703eb as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb 2024-11-13T03:51:34,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb, entries=250, sequenceid=171, filesize=16.5 K 2024-11-13T03:51:34,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469954567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4a84a56d6f7a4e86864c0a8718235c79 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79 2024-11-13T03:51:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T03:51:34,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9342ba649e814b6e804205218009ee87 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87 2024-11-13T03:51:34,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T03:51:34,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 454ms, sequenceid=171, compaction requested=true 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:34,581 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:34,581 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:34,583 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:34,583 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4caff88d38934aadaabfcd49646760a8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=52.2 K 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4caff88d38934aadaabfcd49646760a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:34,583 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:34,583 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0d15c7ff65d9487e90da336756355e59, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=45.2 K 2024-11-13T03:51:34,583 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5339ea3f880d480989f4e1c7da9067c6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731469890799 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d15c7ff65d9487e90da336756355e59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b88ae7ac12ec4350bbc12e8bbfc4ac53, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731469891961 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb9c9c562ac4dbe817cf2d148f4d1cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731469890800 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e394091067fd4a58950e7f838c1703eb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a8bbbcc2254199bb03a9b5f1bbb7df, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731469891961 2024-11-13T03:51:34,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a84a56d6f7a4e86864c0a8718235c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:34,592 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:34,592 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/aa6bdb4812204098b2f5145f8d2c122f is 50, key is test_row_0/A:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,604 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:34,605 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a is 50, key is test_row_0/B:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742104_1280 (size=12493) 2024-11-13T03:51:34,611 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/aa6bdb4812204098b2f5145f8d2c122f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa6bdb4812204098b2f5145f8d2c122f 2024-11-13T03:51:34,617 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into aa6bdb4812204098b2f5145f8d2c122f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:34,617 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:34,617 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=12, startTime=1731469894581; duration=0sec 2024-11-13T03:51:34,617 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:34,617 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:34,617 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:34,620 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:34,620 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:34,620 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:34,620 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/203d4e2e78544d50946dfce60d456795, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=45.2 K 2024-11-13T03:51:34,620 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203d4e2e78544d50946dfce60d456795, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731469889618 2024-11-13T03:51:34,621 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3fb5e7b5e649d1ba2f73aee6180660, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731469890800 2024-11-13T03:51:34,621 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08e63a151053463cb5ee0f5943e49d34, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731469891961 2024-11-13T03:51:34,621 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9342ba649e814b6e804205218009ee87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:34,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742105_1281 (size=12493) 2024-11-13T03:51:34,631 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a 2024-11-13T03:51:34,636 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#239 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:34,637 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/59316c4e359d48dbaa66e81a57817fd8 is 50, key is test_row_0/C:col10/1731469894126/Put/seqid=0 2024-11-13T03:51:34,640 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into e0a1a6a71ff94ebfbe5b57a8eb097c5a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:34,640 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:34,640 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=12, startTime=1731469894581; duration=0sec 2024-11-13T03:51:34,640 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:34,640 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:34,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742106_1282 (size=12493) 2024-11-13T03:51:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T03:51:34,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T03:51:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:34,712 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b8fa8fd5b4314d2186fa9fc8ea231134 is 50, key is test_row_0/A:col10/1731469894247/Put/seqid=0 2024-11-13T03:51:34,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742107_1283 (size=12151) 2024-11-13T03:51:34,721 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b8fa8fd5b4314d2186fa9fc8ea231134 2024-11-13T03:51:34,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5d017e9e70fe4644943e62d1bf2e0ced is 50, key is test_row_0/B:col10/1731469894247/Put/seqid=0 2024-11-13T03:51:34,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742108_1284 (size=12151) 2024-11-13T03:51:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:34,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469954835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T03:51:34,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469954877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469954946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469955017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,019 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469955036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,040 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,048 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/59316c4e359d48dbaa66e81a57817fd8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/59316c4e359d48dbaa66e81a57817fd8 2024-11-13T03:51:35,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469955047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,051 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,053 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 59316c4e359d48dbaa66e81a57817fd8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:35,054 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:35,054 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=12, startTime=1731469894581; duration=0sec 2024-11-13T03:51:35,054 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:35,054 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:35,132 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5d017e9e70fe4644943e62d1bf2e0ced 2024-11-13T03:51:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1663421c238043288f290c68296b87f1 is 50, key is test_row_0/C:col10/1731469894247/Put/seqid=0 2024-11-13T03:51:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742109_1285 (size=12151) 2024-11-13T03:51:35,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1663421c238043288f290c68296b87f1 2024-11-13T03:51:35,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/b8fa8fd5b4314d2186fa9fc8ea231134 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134 2024-11-13T03:51:35,151 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T03:51:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5d017e9e70fe4644943e62d1bf2e0ced as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced 2024-11-13T03:51:35,155 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T03:51:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1663421c238043288f290c68296b87f1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1 2024-11-13T03:51:35,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469955157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T03:51:35,161 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 449ms, sequenceid=209, compaction requested=false 2024-11-13T03:51:35,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:35,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:35,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T03:51:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T03:51:35,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T03:51:35,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 602 msec 2024-11-13T03:51:35,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 607 msec 2024-11-13T03:51:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T03:51:35,187 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:35,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T03:51:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:35,189 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:35,189 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:35,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:35,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:35,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 is 50, key is test_row_0/A:col10/1731469894833/Put/seqid=0 2024-11-13T03:51:35,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742110_1286 (size=12151) 2024-11-13T03:51:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:35,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469895443, max timestamp=1731469895461Num uniques 859; 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469895443, max timestamp=1731469895461Num uniques 859; 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469895443, max timestamp=1731469895461Num uniques 859; 2024-11-13T03:51:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469955503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469955508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469955616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469955616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,751 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 2024-11-13T03:51:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/96ec26104d2e48c4b1ed4100bd11d64c is 50, key is test_row_0/B:col10/1731469894833/Put/seqid=0 2024-11-13T03:51:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742111_1287 (size=12151) 2024-11-13T03:51:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469955827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469955827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469956136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469956137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,165 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/96ec26104d2e48c4b1ed4100bd11d64c 2024-11-13T03:51:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2f742f79d115433c824fbc97e05d41a5 is 50, key is test_row_0/C:col10/1731469894833/Put/seqid=0 2024-11-13T03:51:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742112_1288 (size=12151) 2024-11-13T03:51:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:36,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2f742f79d115433c824fbc97e05d41a5 2024-11-13T03:51:36,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 2024-11-13T03:51:36,587 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T03:51:36,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/96ec26104d2e48c4b1ed4100bd11d64c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c 2024-11-13T03:51:36,592 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T03:51:36,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2f742f79d115433c824fbc97e05d41a5 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5 2024-11-13T03:51:36,597 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T03:51:36,598 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1256ms, sequenceid=221, compaction requested=true 2024-11-13T03:51:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T03:51:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T03:51:36,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T03:51:36,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-13T03:51:36,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4140 sec 2024-11-13T03:51:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:36,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T03:51:36,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:36,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:36,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:36,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:36,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:36,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:36,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/4089946f799e4f2a843eb8ea7821c7b0 is 50, key is test_row_0/A:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:36,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742113_1289 (size=12251) 2024-11-13T03:51:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469956672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469956673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469956787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469956787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469956997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469956997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/4089946f799e4f2a843eb8ea7821c7b0 2024-11-13T03:51:37,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/38998582e4b24c40b1b4de740a467599 is 50, key is test_row_0/B:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:37,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742114_1290 (size=12251) 2024-11-13T03:51:37,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469957306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:37,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469957307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T03:51:37,337 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:37,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T03:51:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:37,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:37,340 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:37,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:37,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/38998582e4b24c40b1b4de740a467599 2024-11-13T03:51:37,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a5aeb87441674a72a6345c661be1c493 is 50, key is test_row_0/C:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:37,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742115_1291 (size=12251) 2024-11-13T03:51:37,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a5aeb87441674a72a6345c661be1c493 2024-11-13T03:51:37,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/4089946f799e4f2a843eb8ea7821c7b0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0 2024-11-13T03:51:37,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T03:51:37,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T03:51:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/38998582e4b24c40b1b4de740a467599 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599 2024-11-13T03:51:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:37,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T03:51:37,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a5aeb87441674a72a6345c661be1c493 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493 2024-11-13T03:51:37,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T03:51:37,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 851ms, sequenceid=259, compaction requested=true 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:37,503 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:37,503 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:37,504 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:37,504 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:37,504 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,504 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa6bdb4812204098b2f5145f8d2c122f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=47.9 K 2024-11-13T03:51:37,504 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:37,504 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6bdb4812204098b2f5145f8d2c122f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:37,505 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:37,505 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,505 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=47.9 K 2024-11-13T03:51:37,505 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8fa8fd5b4314d2186fa9fc8ea231134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731469894230 2024-11-13T03:51:37,505 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a1a6a71ff94ebfbe5b57a8eb097c5a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:37,505 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa67cbc8ba4e4c44900d3dcc2c0b5a79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731469894812 2024-11-13T03:51:37,506 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4089946f799e4f2a843eb8ea7821c7b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:37,506 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d017e9e70fe4644943e62d1bf2e0ced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731469894230 2024-11-13T03:51:37,507 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ec26104d2e48c4b1ed4100bd11d64c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731469894812 2024-11-13T03:51:37,507 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 38998582e4b24c40b1b4de740a467599, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:37,516 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:37,517 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/1129cf1be29b420680d8ebb1cadff1d7 is 50, key is test_row_0/A:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:37,517 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#250 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:37,517 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/6521da637bf3475e8291b35199454663 is 50, key is test_row_0/B:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:37,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742117_1293 (size=12729) 2024-11-13T03:51:37,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742116_1292 (size=12729) 2024-11-13T03:51:37,543 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/6521da637bf3475e8291b35199454663 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/6521da637bf3475e8291b35199454663 2024-11-13T03:51:37,547 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 6521da637bf3475e8291b35199454663(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:37,547 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:37,547 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=12, startTime=1731469897503; duration=0sec 2024-11-13T03:51:37,547 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:37,547 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:37,548 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:51:37,549 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:51:37,549 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:37,549 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,549 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/59316c4e359d48dbaa66e81a57817fd8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=47.9 K 2024-11-13T03:51:37,549 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 59316c4e359d48dbaa66e81a57817fd8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469894117 2024-11-13T03:51:37,550 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1663421c238043288f290c68296b87f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731469894230 2024-11-13T03:51:37,550 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f742f79d115433c824fbc97e05d41a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731469894812 2024-11-13T03:51:37,550 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a5aeb87441674a72a6345c661be1c493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:37,557 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:37,558 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/5b43aa5f161b4adcb4a042c5e267892e is 50, key is test_row_0/C:col10/1731469895507/Put/seqid=0 2024-11-13T03:51:37,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742118_1294 (size=12729) 2024-11-13T03:51:37,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:37,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/9f0aecc8b43f4090982397cb5cf20c53 is 50, key is test_row_0/A:col10/1731469896670/Put/seqid=0 2024-11-13T03:51:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742119_1295 (size=12301) 2024-11-13T03:51:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:37,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469897848, max timestamp=1731469897857Num uniques 859; 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469897848, max timestamp=1731469897857Num uniques 859; 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469897848, max timestamp=1731469897857Num uniques 859; 2024-11-13T03:51:37,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:51:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469957900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469957905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:37,942 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/1129cf1be29b420680d8ebb1cadff1d7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/1129cf1be29b420680d8ebb1cadff1d7 2024-11-13T03:51:37,946 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 1129cf1be29b420680d8ebb1cadff1d7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:37,946 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:37,946 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=12, startTime=1731469897503; duration=0sec 2024-11-13T03:51:37,946 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:37,946 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:37,967 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/5b43aa5f161b4adcb4a042c5e267892e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/5b43aa5f161b4adcb4a042c5e267892e 2024-11-13T03:51:37,972 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 5b43aa5f161b4adcb4a042c5e267892e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:37,972 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:37,972 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=12, startTime=1731469897503; duration=0sec 2024-11-13T03:51:37,972 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:37,972 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469958007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469958017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,057 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/9f0aecc8b43f4090982397cb5cf20c53 2024-11-13T03:51:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/187078b026e74d8aa642453ba318c861 is 50, key is test_row_0/B:col10/1731469896670/Put/seqid=0 2024-11-13T03:51:38,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742120_1296 (size=12301) 2024-11-13T03:51:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469958216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469958227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/187078b026e74d8aa642453ba318c861 2024-11-13T03:51:38,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/e30ab289fddd4205a9f6e211e83be644 is 50, key is test_row_0/C:col10/1731469896670/Put/seqid=0 2024-11-13T03:51:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742121_1297 (size=12301) 2024-11-13T03:51:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:38,477 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/e30ab289fddd4205a9f6e211e83be644 2024-11-13T03:51:38,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/9f0aecc8b43f4090982397cb5cf20c53 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53 2024-11-13T03:51:38,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:51:38,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/187078b026e74d8aa642453ba318c861 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861 2024-11-13T03:51:38,488 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:51:38,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/e30ab289fddd4205a9f6e211e83be644 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644 2024-11-13T03:51:38,491 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T03:51:38,492 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 847ms, sequenceid=269, compaction requested=false 2024-11-13T03:51:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T03:51:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T03:51:38,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T03:51:38,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-13T03:51:38,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1580 sec 2024-11-13T03:51:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:38,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:51:38,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/62f9ec42481b48d19526016fab466b07 is 50, key is test_row_0/A:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:38,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742122_1298 (size=14741) 2024-11-13T03:51:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469958545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469958545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469958657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469958657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469958867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469958867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:38,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/62f9ec42481b48d19526016fab466b07 2024-11-13T03:51:38,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4ee949b07b3c465e9f08ebb38d04dc88 is 50, key is test_row_0/B:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:38,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742123_1299 (size=12301) 2024-11-13T03:51:39,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469959176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:39,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469959176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:39,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4ee949b07b3c465e9f08ebb38d04dc88 2024-11-13T03:51:39,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2c54825baa664f7b927ab316963a943c is 50, key is test_row_0/C:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:39,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742124_1300 (size=12301) 2024-11-13T03:51:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T03:51:39,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:39,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T03:51:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T03:51:39,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:39,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:39,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T03:51:39,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T03:51:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469959686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:39,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469959686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:39,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2c54825baa664f7b927ab316963a943c 2024-11-13T03:51:39,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/62f9ec42481b48d19526016fab466b07 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07 2024-11-13T03:51:39,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07, entries=200, sequenceid=309, filesize=14.4 K 2024-11-13T03:51:39,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/4ee949b07b3c465e9f08ebb38d04dc88 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88 2024-11-13T03:51:39,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T03:51:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/2c54825baa664f7b927ab316963a943c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c 2024-11-13T03:51:39,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T03:51:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T03:51:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:39,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1267ms, sequenceid=309, compaction requested=true 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:39,797 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:39,797 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:39,798 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,798 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,798 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/6521da637bf3475e8291b35199454663, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.5 K 2024-11-13T03:51:39,798 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/1129cf1be29b420680d8ebb1cadff1d7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=38.8 K 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6521da637bf3475e8291b35199454663, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:39,798 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1129cf1be29b420680d8ebb1cadff1d7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:39,799 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 187078b026e74d8aa642453ba318c861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469896664 2024-11-13T03:51:39,799 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0aecc8b43f4090982397cb5cf20c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469896664 2024-11-13T03:51:39,799 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f9ec42481b48d19526016fab466b07, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:39,799 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee949b07b3c465e9f08ebb38d04dc88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:39,805 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:39,806 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/021815b66de14cd08414cafd30f13b0c is 50, key is test_row_0/B:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:39,808 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:39,809 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/92a021b0fc6747a3bf8c6b7cf2745d79 is 50, key is test_row_0/A:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T03:51:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742125_1301 (size=12881) 2024-11-13T03:51:39,817 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/021815b66de14cd08414cafd30f13b0c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/021815b66de14cd08414cafd30f13b0c 2024-11-13T03:51:39,822 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 021815b66de14cd08414cafd30f13b0c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:39,822 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:39,822 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469899797; duration=0sec 2024-11-13T03:51:39,822 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:39,822 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:39,823 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:39,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742126_1302 (size=12881) 2024-11-13T03:51:39,825 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:39,825 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:39,825 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,825 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/5b43aa5f161b4adcb4a042c5e267892e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.5 K 2024-11-13T03:51:39,825 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b43aa5f161b4adcb4a042c5e267892e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731469895507 2024-11-13T03:51:39,826 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e30ab289fddd4205a9f6e211e83be644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731469896664 2024-11-13T03:51:39,826 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c54825baa664f7b927ab316963a943c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:39,828 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/92a021b0fc6747a3bf8c6b7cf2745d79 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/92a021b0fc6747a3bf8c6b7cf2745d79 2024-11-13T03:51:39,833 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:39,834 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/cf45d090e8d745938f92537da4c79bdb is 50, key is test_row_0/C:col10/1731469897904/Put/seqid=0 2024-11-13T03:51:39,834 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 92a021b0fc6747a3bf8c6b7cf2745d79(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:39,834 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:39,834 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469899797; duration=0sec 2024-11-13T03:51:39,834 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:39,835 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:39,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742127_1303 (size=12881) 2024-11-13T03:51:39,845 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/cf45d090e8d745938f92537da4c79bdb as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/cf45d090e8d745938f92537da4c79bdb 2024-11-13T03:51:39,850 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into cf45d090e8d745938f92537da4c79bdb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:39,850 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:39,850 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469899797; duration=0sec 2024-11-13T03:51:39,850 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:39,850 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:39,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T03:51:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:39,948 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d266830e06624986ab7c4d9627bec922 is 50, key is test_row_1/A:col10/1731469898543/Put/seqid=0 2024-11-13T03:51:39,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742128_1304 (size=9857) 2024-11-13T03:51:39,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d266830e06624986ab7c4d9627bec922 2024-11-13T03:51:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/eeb8a378495e4ce09ad3fab1489e3173 is 50, key is test_row_1/B:col10/1731469898543/Put/seqid=0 2024-11-13T03:51:39,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742129_1305 (size=9857) 2024-11-13T03:51:39,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/eeb8a378495e4ce09ad3fab1489e3173 2024-11-13T03:51:39,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8fc016bd8514457780d171fac8bf9c0f is 50, key is test_row_1/C:col10/1731469898543/Put/seqid=0 2024-11-13T03:51:39,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742130_1306 (size=9857) 2024-11-13T03:51:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T03:51:40,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8fc016bd8514457780d171fac8bf9c0f 2024-11-13T03:51:40,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/d266830e06624986ab7c4d9627bec922 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922 2024-11-13T03:51:40,393 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922, entries=100, sequenceid=320, filesize=9.6 K 2024-11-13T03:51:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/eeb8a378495e4ce09ad3fab1489e3173 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173 2024-11-13T03:51:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173, entries=100, sequenceid=320, filesize=9.6 K 2024-11-13T03:51:40,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/8fc016bd8514457780d171fac8bf9c0f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f 2024-11-13T03:51:40,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f, entries=100, sequenceid=320, filesize=9.6 K 2024-11-13T03:51:40,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 460ms, sequenceid=320, compaction requested=false 2024-11-13T03:51:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T03:51:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T03:51:40,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T03:51:40,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 920 msec 2024-11-13T03:51:40,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 925 msec 2024-11-13T03:51:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T03:51:40,627 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:40,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-13T03:51:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:40,629 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:40,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:40,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:40,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:40,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/98eb2f9405304bb4bc802845a2fc9c36 is 50, key is test_row_0/A:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742131_1307 (size=14741) 2024-11-13T03:51:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469900731, max timestamp=1731469900739Num uniques 859; 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469900731, max timestamp=1731469900739Num uniques 859; 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469900731, max timestamp=1731469900739Num uniques 859; 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:40,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:40,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:40,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469960786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469960791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469960897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:40,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469960906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:40,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:40,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:41,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469961106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469961117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/98eb2f9405304bb4bc802845a2fc9c36 2024-11-13T03:51:41,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/9fb9bb5bab6342858449ae1fbfb57090 is 50, key is test_row_0/B:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:41,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742132_1308 (size=12301) 2024-11-13T03:51:41,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:41,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469961417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469961427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/9fb9bb5bab6342858449ae1fbfb57090 2024-11-13T03:51:41,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/700e0813a0904f20872b43b96e44f5b6 is 50, key is test_row_0/C:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:41,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742133_1309 (size=12301) 2024-11-13T03:51:41,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:41,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:41,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469961927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469961937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:41,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/700e0813a0904f20872b43b96e44f5b6 2024-11-13T03:51:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:41,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/98eb2f9405304bb4bc802845a2fc9c36 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36 2024-11-13T03:51:41,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36, entries=200, sequenceid=330, filesize=14.4 K 2024-11-13T03:51:41,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/9fb9bb5bab6342858449ae1fbfb57090 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090 2024-11-13T03:51:41,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T03:51:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/700e0813a0904f20872b43b96e44f5b6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6 2024-11-13T03:51:41,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T03:51:41,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1252ms, sequenceid=330, compaction requested=true 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:41,956 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:41,956 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:41,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:41,957 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37479 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:41,957 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:41,957 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:41,957 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:41,957 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,957 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,957 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/021815b66de14cd08414cafd30f13b0c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=34.2 K 2024-11-13T03:51:41,957 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/92a021b0fc6747a3bf8c6b7cf2745d79, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.6 K 2024-11-13T03:51:41,958 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 021815b66de14cd08414cafd30f13b0c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:41,958 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a021b0fc6747a3bf8c6b7cf2745d79, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:41,958 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb8a378495e4ce09ad3fab1489e3173, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731469898543 2024-11-13T03:51:41,958 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb9bb5bab6342858449ae1fbfb57090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:41,959 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d266830e06624986ab7c4d9627bec922, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731469898543 2024-11-13T03:51:41,959 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98eb2f9405304bb4bc802845a2fc9c36, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:41,965 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:41,965 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/3560f53ad5e743758ae501a5605d0cda is 50, key is test_row_0/B:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:41,969 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:41,969 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/5d8dd6df99924348a3e40bf602d40ed2 is 50, key is test_row_0/A:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:41,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742134_1310 (size=12983) 2024-11-13T03:51:41,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742135_1311 (size=12983) 2024-11-13T03:51:41,980 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/5d8dd6df99924348a3e40bf602d40ed2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5d8dd6df99924348a3e40bf602d40ed2 2024-11-13T03:51:41,985 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 5d8dd6df99924348a3e40bf602d40ed2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:41,986 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:41,986 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469901956; duration=0sec 2024-11-13T03:51:41,986 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:41,986 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:41,986 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:41,987 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:41,987 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:41,987 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:41,987 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/cf45d090e8d745938f92537da4c79bdb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=34.2 K 2024-11-13T03:51:41,987 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf45d090e8d745938f92537da4c79bdb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731469897892 2024-11-13T03:51:41,987 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc016bd8514457780d171fac8bf9c0f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731469898543 2024-11-13T03:51:41,989 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700e0813a0904f20872b43b96e44f5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:41,995 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:41,996 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9980bd294896464b96890b977b893f9c is 50, key is test_row_0/C:col10/1731469900700/Put/seqid=0 2024-11-13T03:51:42,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742136_1312 (size=12983) 2024-11-13T03:51:42,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:42,009 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:42,011 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9980bd294896464b96890b977b893f9c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9980bd294896464b96890b977b893f9c 2024-11-13T03:51:42,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/43452f62bbc9484699ffbce37ffde0b4 is 50, key is test_row_0/A:col10/1731469900776/Put/seqid=0 2024-11-13T03:51:42,016 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 9980bd294896464b96890b977b893f9c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:42,016 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:42,016 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469901956; duration=0sec 2024-11-13T03:51:42,016 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:42,016 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742137_1313 (size=12301) 2024-11-13T03:51:42,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/43452f62bbc9484699ffbce37ffde0b4 2024-11-13T03:51:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5c863055423245bf8a281f08e35a2343 is 50, key is test_row_0/B:col10/1731469900776/Put/seqid=0 2024-11-13T03:51:42,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742138_1314 (size=12301) 2024-11-13T03:51:42,031 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5c863055423245bf8a281f08e35a2343 2024-11-13T03:51:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/c60ffaedd6b0406cb9f89bedf1545d52 is 50, key is test_row_0/C:col10/1731469900776/Put/seqid=0 2024-11-13T03:51:42,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742139_1315 (size=12301) 2024-11-13T03:51:42,376 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/3560f53ad5e743758ae501a5605d0cda as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/3560f53ad5e743758ae501a5605d0cda 2024-11-13T03:51:42,379 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 3560f53ad5e743758ae501a5605d0cda(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:42,379 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:42,379 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469901956; duration=0sec 2024-11-13T03:51:42,380 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:42,380 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:42,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/c60ffaedd6b0406cb9f89bedf1545d52 2024-11-13T03:51:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/43452f62bbc9484699ffbce37ffde0b4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4 2024-11-13T03:51:42,448 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4, entries=150, sequenceid=368, filesize=12.0 K 2024-11-13T03:51:42,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/5c863055423245bf8a281f08e35a2343 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343 2024-11-13T03:51:42,453 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343, entries=150, sequenceid=368, filesize=12.0 K 2024-11-13T03:51:42,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/c60ffaedd6b0406cb9f89bedf1545d52 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52 2024-11-13T03:51:42,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52, entries=150, sequenceid=368, filesize=12.0 K 2024-11-13T03:51:42,458 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 449ms, sequenceid=368, compaction requested=false 2024-11-13T03:51:42,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:42,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:42,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-13T03:51:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-13T03:51:42,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T03:51:42,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-13T03:51:42,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8340 sec 2024-11-13T03:51:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T03:51:42,777 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:42,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-13T03:51:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T03:51:42,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:42,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:42,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T03:51:42,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T03:51:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-13T03:51:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-13T03:51:42,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T03:51:42,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-13T03:51:42,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 162 msec 2024-11-13T03:51:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:42,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:51:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:42,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:42,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:42,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/be2300ca84ae4d2f8aca31e8eaab6258 is 50, key is test_row_0/A:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:42,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742140_1316 (size=12301) 2024-11-13T03:51:42,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/be2300ca84ae4d2f8aca31e8eaab6258 2024-11-13T03:51:42,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0764acb77f9c452a82d78580c5597a3b is 50, key is test_row_0/B:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:42,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742141_1317 (size=12301) 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469902962, max timestamp=1731469902984Num uniques 859; 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469902962, max timestamp=1731469902984Num uniques 859; 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469902962, max timestamp=1731469902984Num uniques 859; 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:42,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:51:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469963020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469963019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T03:51:43,097 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:43,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-13T03:51:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:43,100 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:43,100 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:43,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469963127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469963127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:43,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T03:51:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469963337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469963337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0764acb77f9c452a82d78580c5597a3b 2024-11-13T03:51:43,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1ce579cab9554b78b8eb1881d8009bc9 is 50, key is test_row_0/C:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:43,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742142_1318 (size=12301) 2024-11-13T03:51:43,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T03:51:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:43,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T03:51:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469963646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469963646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:43,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T03:51:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:43,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1ce579cab9554b78b8eb1881d8009bc9 2024-11-13T03:51:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/be2300ca84ae4d2f8aca31e8eaab6258 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258 2024-11-13T03:51:43,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258, entries=150, sequenceid=381, filesize=12.0 K 2024-11-13T03:51:43,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0764acb77f9c452a82d78580c5597a3b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b 2024-11-13T03:51:43,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b, entries=150, sequenceid=381, filesize=12.0 K 2024-11-13T03:51:43,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/1ce579cab9554b78b8eb1881d8009bc9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9 2024-11-13T03:51:43,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9, entries=150, sequenceid=381, filesize=12.0 K 2024-11-13T03:51:43,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 860ms, sequenceid=381, compaction requested=true 2024-11-13T03:51:43,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:43,808 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:43,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:43,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:43,808 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:43,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:43,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:43,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:43,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:43,809 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:43,810 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:43,810 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,810 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5d8dd6df99924348a3e40bf602d40ed2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.7 K 2024-11-13T03:51:43,810 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:43,810 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:43,810 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,810 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/3560f53ad5e743758ae501a5605d0cda, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.7 K 2024-11-13T03:51:43,811 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d8dd6df99924348a3e40bf602d40ed2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:43,811 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3560f53ad5e743758ae501a5605d0cda, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:43,811 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43452f62bbc9484699ffbce37ffde0b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731469900776 2024-11-13T03:51:43,811 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c863055423245bf8a281f08e35a2343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731469900776 2024-11-13T03:51:43,812 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting be2300ca84ae4d2f8aca31e8eaab6258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:43,812 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0764acb77f9c452a82d78580c5597a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:43,820 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:43,820 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/c900c19f8a7d47258b77e358b2c27dd9 is 50, key is test_row_0/B:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:43,823 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:43,824 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/a492fbcf6aa1475ba354263c4d96d262 is 50, key is test_row_0/A:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742144_1320 (size=13085) 2024-11-13T03:51:43,856 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/a492fbcf6aa1475ba354263c4d96d262 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/a492fbcf6aa1475ba354263c4d96d262 2024-11-13T03:51:43,861 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into a492fbcf6aa1475ba354263c4d96d262(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:43,861 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:43,861 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469903808; duration=0sec 2024-11-13T03:51:43,861 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:43,862 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:43,862 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:43,863 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:43,863 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:43,863 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,863 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9980bd294896464b96890b977b893f9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.7 K 2024-11-13T03:51:43,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T03:51:43,864 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9980bd294896464b96890b977b893f9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731469900697 2024-11-13T03:51:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:43,864 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T03:51:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:43,864 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60ffaedd6b0406cb9f89bedf1545d52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731469900776 2024-11-13T03:51:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:51:43,865 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce579cab9554b78b8eb1881d8009bc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:43,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742143_1319 (size=13085) 2024-11-13T03:51:43,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/8db9c8bd480a4d75b3b985ec506dc930 is 50, key is test_row_0/A:col10/1731469903018/Put/seqid=0 2024-11-13T03:51:43,884 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:43,885 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/23bad0029fa44b669ff79bedaf169aa9 is 50, key is test_row_0/C:col10/1731469902941/Put/seqid=0 2024-11-13T03:51:43,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742145_1321 (size=12301) 2024-11-13T03:51:43,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742146_1322 (size=13085) 2024-11-13T03:51:43,932 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/23bad0029fa44b669ff79bedaf169aa9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/23bad0029fa44b669ff79bedaf169aa9 2024-11-13T03:51:43,938 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 23bad0029fa44b669ff79bedaf169aa9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:43,938 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:43,938 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469903809; duration=0sec 2024-11-13T03:51:43,938 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:43,938 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:44,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:44,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469964173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469964180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:44,273 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/c900c19f8a7d47258b77e358b2c27dd9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c900c19f8a7d47258b77e358b2c27dd9 2024-11-13T03:51:44,277 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into c900c19f8a7d47258b77e358b2c27dd9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:44,277 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:44,277 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469903808; duration=0sec 2024-11-13T03:51:44,277 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:44,278 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:44,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469964286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469964286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/8db9c8bd480a4d75b3b985ec506dc930 2024-11-13T03:51:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/b9a9b2b0f9d3429bbe63a5734735b1ea is 50, key is test_row_0/B:col10/1731469903018/Put/seqid=0 2024-11-13T03:51:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742147_1323 (size=12301) 2024-11-13T03:51:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469964497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469964497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,730 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/b9a9b2b0f9d3429bbe63a5734735b1ea 2024-11-13T03:51:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9534b0f6712a4cfda27edefdde4a8118 is 50, key is test_row_0/C:col10/1731469903018/Put/seqid=0 2024-11-13T03:51:44,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742148_1324 (size=12301) 2024-11-13T03:51:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469964806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469964806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55538 deadline: 1731469965107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,110 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18302 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55510 deadline: 1731469965117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,119 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18313 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9534b0f6712a4cfda27edefdde4a8118 2024-11-13T03:51:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/8db9c8bd480a4d75b3b985ec506dc930 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930 2024-11-13T03:51:45,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T03:51:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/b9a9b2b0f9d3429bbe63a5734735b1ea as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea 2024-11-13T03:51:45,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55494 deadline: 1731469965146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,150 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in e3e30c23eeb6b0b0afd7668c5b9a55d3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18342 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:51:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T03:51:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/9534b0f6712a4cfda27edefdde4a8118 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118 2024-11-13T03:51:45,157 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T03:51:45,157 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1293ms, sequenceid=419, compaction requested=false 2024-11-13T03:51:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-13T03:51:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-13T03:51:45,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T03:51:45,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-11-13T03:51:45,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0640 sec 2024-11-13T03:51:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T03:51:45,256 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:45,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:51:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-13T03:51:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:45,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:51:45,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:51:45,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:51:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:45,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:45,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/ca4c550de9334966acff859ccc5824f9 is 50, key is test_row_0/A:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:45,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742149_1325 (size=17181) 2024-11-13T03:51:45,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469965356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469965360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:45,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:45,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469965466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469965466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,561 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:51:45,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469965677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469965677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/ca4c550de9334966acff859ccc5824f9 2024-11-13T03:51:45,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/30b16589e41d4684916cae293dbac9cf is 50, key is test_row_0/B:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742150_1326 (size=12301) 2024-11-13T03:51:45,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:45,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469965987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469965987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:46,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,098 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-13T03:51:46,098 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:46,098 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,098 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:46,100 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-13T03:51:46,100 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:46,100 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:46,100 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,101 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-13T03:51:46,102 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:46,102 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,102 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:46,102 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-13T03:51:46,102 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:46,102 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,102 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:46,104 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-13T03:51:46,104 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:46,104 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,104 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:46,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:46,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/30b16589e41d4684916cae293dbac9cf 2024-11-13T03:51:46,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a40f0a7be2274b13b424b7f1547ed5d7 is 50, key is test_row_0/C:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:46,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742151_1327 (size=12301) 2024-11-13T03:51:46,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:46,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:46,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:46,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:51:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:51:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55516 deadline: 1731469966497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55526 deadline: 1731469966498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:51:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:51:46,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a40f0a7be2274b13b424b7f1547ed5d7 2024-11-13T03:51:46,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/ca4c550de9334966acff859ccc5824f9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9 2024-11-13T03:51:46,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9, entries=250, sequenceid=433, filesize=16.8 K 2024-11-13T03:51:46,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/30b16589e41d4684916cae293dbac9cf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf 2024-11-13T03:51:46,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T03:51:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a40f0a7be2274b13b424b7f1547ed5d7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7 2024-11-13T03:51:46,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7, entries=150, sequenceid=433, filesize=12.0 K 2024-11-13T03:51:46,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1260ms, sequenceid=433, compaction requested=true 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:51:46,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e3e30c23eeb6b0b0afd7668c5b9a55d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:51:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:46,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:46,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:46,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:46,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/A is initiating minor compaction (all files) 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/B is initiating minor compaction (all files) 2024-11-13T03:51:46,580 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/A in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,580 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/B in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,580 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/a492fbcf6aa1475ba354263c4d96d262, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=41.6 K 2024-11-13T03:51:46,580 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c900c19f8a7d47258b77e358b2c27dd9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.8 K 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting a492fbcf6aa1475ba354263c4d96d262, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c900c19f8a7d47258b77e358b2c27dd9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db9c8bd480a4d75b3b985ec506dc930, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731469903015 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a9b2b0f9d3429bbe63a5734735b1ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731469903015 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4c550de9334966acff859ccc5824f9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731469904159 2024-11-13T03:51:46,580 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b16589e41d4684916cae293dbac9cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731469904177 2024-11-13T03:51:46,588 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#B#compaction#285 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:46,588 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0625642f16f6427baeba4d2f29686f86 is 50, key is test_row_0/B:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:46,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742152_1328 (size=13187) 2024-11-13T03:51:46,606 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#A#compaction#286 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:46,606 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/2e1c59323d504cfdb96a03d4e494a717 is 50, key is test_row_0/A:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:46,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742153_1329 (size=13187) 2024-11-13T03:51:46,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-13T03:51:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:46,640 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:46,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/21da9c2a82d54fd49233f2e78f1492db is 50, key is test_row_0/A:col10/1731469905359/Put/seqid=0 2024-11-13T03:51:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742154_1330 (size=12301) 2024-11-13T03:51:47,004 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/0625642f16f6427baeba4d2f29686f86 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0625642f16f6427baeba4d2f29686f86 2024-11-13T03:51:47,010 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/B of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 0625642f16f6427baeba4d2f29686f86(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:47,010 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:47,010 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/B, priority=13, startTime=1731469906579; duration=0sec 2024-11-13T03:51:47,010 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:51:47,010 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:B 2024-11-13T03:51:47,010 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:51:47,011 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:51:47,011 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): e3e30c23eeb6b0b0afd7668c5b9a55d3/C is initiating minor compaction (all files) 2024-11-13T03:51:47,011 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e3e30c23eeb6b0b0afd7668c5b9a55d3/C in TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:47,011 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/23bad0029fa44b669ff79bedaf169aa9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp, totalSize=36.8 K 2024-11-13T03:51:47,012 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bad0029fa44b669ff79bedaf169aa9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731469902941 2024-11-13T03:51:47,012 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9534b0f6712a4cfda27edefdde4a8118, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731469903015 2024-11-13T03:51:47,013 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a40f0a7be2274b13b424b7f1547ed5d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731469904177 2024-11-13T03:51:47,015 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/2e1c59323d504cfdb96a03d4e494a717 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/2e1c59323d504cfdb96a03d4e494a717 2024-11-13T03:51:47,019 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/A of e3e30c23eeb6b0b0afd7668c5b9a55d3 into 2e1c59323d504cfdb96a03d4e494a717(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:47,020 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:47,020 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/A, priority=13, startTime=1731469906579; duration=0sec 2024-11-13T03:51:47,020 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:47,020 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:A 2024-11-13T03:51:47,023 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e3e30c23eeb6b0b0afd7668c5b9a55d3#C#compaction#288 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:51:47,023 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a98bb1ad3f88477ea19a447be72b8ca3 is 50, key is test_row_0/C:col10/1731469905317/Put/seqid=0 2024-11-13T03:51:47,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742155_1331 (size=13187) 2024-11-13T03:51:47,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/21da9c2a82d54fd49233f2e78f1492db 2024-11-13T03:51:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/1a0dadbb2e3747a29e82a5ab7cc094c2 is 50, key is test_row_0/B:col10/1731469905359/Put/seqid=0 2024-11-13T03:51:47,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742156_1332 (size=12301) 2024-11-13T03:51:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:47,438 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/a98bb1ad3f88477ea19a447be72b8ca3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a98bb1ad3f88477ea19a447be72b8ca3 2024-11-13T03:51:47,443 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e3e30c23eeb6b0b0afd7668c5b9a55d3/C of e3e30c23eeb6b0b0afd7668c5b9a55d3 into a98bb1ad3f88477ea19a447be72b8ca3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:51:47,443 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:47,443 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., storeName=e3e30c23eeb6b0b0afd7668c5b9a55d3/C, priority=13, startTime=1731469906579; duration=0sec 2024-11-13T03:51:47,443 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:51:47,443 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e3e30c23eeb6b0b0afd7668c5b9a55d3:C 2024-11-13T03:51:47,462 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/1a0dadbb2e3747a29e82a5ab7cc094c2 2024-11-13T03:51:47,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/141d747c8dcb483fad08b84c2eecbece is 50, key is test_row_0/C:col10/1731469905359/Put/seqid=0 2024-11-13T03:51:47,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742157_1333 (size=12301) 2024-11-13T03:51:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:47,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. as already flushing 2024-11-13T03:51:47,508 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-13T03:51:47,508 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:47,508 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:47,509 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:47,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:47,519 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-13T03:51:47,519 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:47,519 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:47,519 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:47,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:47,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/141d747c8dcb483fad08b84c2eecbece 2024-11-13T03:51:47,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/21da9c2a82d54fd49233f2e78f1492db as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/21da9c2a82d54fd49233f2e78f1492db 2024-11-13T03:51:47,895 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/21da9c2a82d54fd49233f2e78f1492db, entries=150, sequenceid=453, filesize=12.0 K 2024-11-13T03:51:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/1a0dadbb2e3747a29e82a5ab7cc094c2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/1a0dadbb2e3747a29e82a5ab7cc094c2 2024-11-13T03:51:47,901 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/1a0dadbb2e3747a29e82a5ab7cc094c2, entries=150, sequenceid=453, filesize=12.0 K 2024-11-13T03:51:47,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/141d747c8dcb483fad08b84c2eecbece as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/141d747c8dcb483fad08b84c2eecbece 2024-11-13T03:51:47,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/141d747c8dcb483fad08b84c2eecbece, entries=150, sequenceid=453, filesize=12.0 K 2024-11-13T03:51:47,909 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1269ms, sequenceid=453, compaction requested=false 2024-11-13T03:51:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: 2024-11-13T03:51:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-13T03:51:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-13T03:51:47,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-13T03:51:47,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6520 sec 2024-11-13T03:51:47,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.6560 sec 2024-11-13T03:51:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T03:51:49,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:55,142 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-13T03:51:55,142 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:55,143 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,143 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:55,160 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-13T03:51:55,160 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:55,160 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,161 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:55,210 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-13T03:51:55,210 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:51:55,210 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,211 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:55,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:51:55,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-13T03:51:55,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 179 2024-11-13T03:51:55,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3807 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3428 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3786 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3821 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3427 2024-11-13T03:51:55,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:51:55,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:51:55,212 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:51:55,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:55,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:51:55,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:51:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:55,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469915220"}]},"ts":"1731469915220"} 2024-11-13T03:51:55,221 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:51:55,221 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:51:55,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:51:55,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, UNASSIGN}] 2024-11-13T03:51:55,225 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, UNASSIGN 2024-11-13T03:51:55,225 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=e3e30c23eeb6b0b0afd7668c5b9a55d3, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:55,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, UNASSIGN because future has completed 2024-11-13T03:51:55,227 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:51:55,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:51:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:55,380 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:55,380 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:51:55,380 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing e3e30c23eeb6b0b0afd7668c5b9a55d3, disabling compactions & flushes 2024-11-13T03:51:55,381 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:55,381 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:55,381 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. after waiting 0 ms 2024-11-13T03:51:55,381 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:55,381 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=A 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=B 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e3e30c23eeb6b0b0afd7668c5b9a55d3, store=C 2024-11-13T03:51:55,382 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:51:55,395 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/cee0c32c3fcc4a178fd7a1aa90e34440 is 50, key is test_row_0/A:col10/1731469915208/Put/seqid=0 2024-11-13T03:51:55,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742158_1334 (size=12301) 2024-11-13T03:51:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:55,800 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/cee0c32c3fcc4a178fd7a1aa90e34440 2024-11-13T03:51:55,815 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/87a0553b5d5d4a62a41e582b89d72145 is 50, key is test_row_0/B:col10/1731469915208/Put/seqid=0 2024-11-13T03:51:55,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742159_1335 (size=12301) 2024-11-13T03:51:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:56,221 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/87a0553b5d5d4a62a41e582b89d72145 2024-11-13T03:51:56,235 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6da822e0922d463fab6bf4cf50ce39ac is 50, key is test_row_0/C:col10/1731469915208/Put/seqid=0 2024-11-13T03:51:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742160_1336 (size=12301) 2024-11-13T03:51:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:56,640 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6da822e0922d463fab6bf4cf50ce39ac 2024-11-13T03:51:56,650 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/A/cee0c32c3fcc4a178fd7a1aa90e34440 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/cee0c32c3fcc4a178fd7a1aa90e34440 2024-11-13T03:51:56,655 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/cee0c32c3fcc4a178fd7a1aa90e34440, entries=150, sequenceid=464, filesize=12.0 K 2024-11-13T03:51:56,656 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/B/87a0553b5d5d4a62a41e582b89d72145 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/87a0553b5d5d4a62a41e582b89d72145 2024-11-13T03:51:56,658 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/87a0553b5d5d4a62a41e582b89d72145, entries=150, sequenceid=464, filesize=12.0 K 2024-11-13T03:51:56,659 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/.tmp/C/6da822e0922d463fab6bf4cf50ce39ac as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6da822e0922d463fab6bf4cf50ce39ac 2024-11-13T03:51:56,662 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6da822e0922d463fab6bf4cf50ce39ac, entries=150, sequenceid=464, filesize=12.0 K 2024-11-13T03:51:56,662 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1281ms, sequenceid=464, compaction requested=true 2024-11-13T03:51:56,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d63afa8475c344adb6045e49a6bb312d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4caff88d38934aadaabfcd49646760a8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa6bdb4812204098b2f5145f8d2c122f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/1129cf1be29b420680d8ebb1cadff1d7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/92a021b0fc6747a3bf8c6b7cf2745d79, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5d8dd6df99924348a3e40bf602d40ed2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/a492fbcf6aa1475ba354263c4d96d262, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9] to archive 2024-11-13T03:51:56,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:56,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/bb4dd0f0f8f04656b14fd7ea4714e36e 2024-11-13T03:51:56,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/90b01ee7dff74326a356889438bad06b 2024-11-13T03:51:56,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/17a6af06db5f4d91bf837bed9894e854 2024-11-13T03:51:56,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d63afa8475c344adb6045e49a6bb312d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d63afa8475c344adb6045e49a6bb312d 2024-11-13T03:51:56,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/95dfedde23cb4632ab2c7751d28ef3c3 2024-11-13T03:51:56,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4caff88d38934aadaabfcd49646760a8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4caff88d38934aadaabfcd49646760a8 2024-11-13T03:51:56,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d981d334406542c5a088ca2e0c519a3e 2024-11-13T03:51:56,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5339ea3f880d480989f4e1c7da9067c6 2024-11-13T03:51:56,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b88ae7ac12ec4350bbc12e8bbfc4ac53 2024-11-13T03:51:56,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/e394091067fd4a58950e7f838c1703eb 2024-11-13T03:51:56,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa6bdb4812204098b2f5145f8d2c122f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa6bdb4812204098b2f5145f8d2c122f 2024-11-13T03:51:56,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/b8fa8fd5b4314d2186fa9fc8ea231134 2024-11-13T03:51:56,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/aa67cbc8ba4e4c44900d3dcc2c0b5a79 2024-11-13T03:51:56,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/1129cf1be29b420680d8ebb1cadff1d7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/1129cf1be29b420680d8ebb1cadff1d7 2024-11-13T03:51:56,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/4089946f799e4f2a843eb8ea7821c7b0 2024-11-13T03:51:56,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/9f0aecc8b43f4090982397cb5cf20c53 2024-11-13T03:51:56,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/62f9ec42481b48d19526016fab466b07 2024-11-13T03:51:56,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/92a021b0fc6747a3bf8c6b7cf2745d79 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/92a021b0fc6747a3bf8c6b7cf2745d79 2024-11-13T03:51:56,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/d266830e06624986ab7c4d9627bec922 2024-11-13T03:51:56,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/98eb2f9405304bb4bc802845a2fc9c36 2024-11-13T03:51:56,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5d8dd6df99924348a3e40bf602d40ed2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/5d8dd6df99924348a3e40bf602d40ed2 2024-11-13T03:51:56,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/43452f62bbc9484699ffbce37ffde0b4 2024-11-13T03:51:56,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/a492fbcf6aa1475ba354263c4d96d262 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/a492fbcf6aa1475ba354263c4d96d262 2024-11-13T03:51:56,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/be2300ca84ae4d2f8aca31e8eaab6258 2024-11-13T03:51:56,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/8db9c8bd480a4d75b3b985ec506dc930 2024-11-13T03:51:56,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/ca4c550de9334966acff859ccc5824f9 2024-11-13T03:51:56,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/66b244112eda41748b9fddd33f0aff44, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0d15c7ff65d9487e90da336756355e59, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/6521da637bf3475e8291b35199454663, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/021815b66de14cd08414cafd30f13b0c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/3560f53ad5e743758ae501a5605d0cda, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c900c19f8a7d47258b77e358b2c27dd9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf] to archive 2024-11-13T03:51:56,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:56,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bf881d00f5ac4ec08977bae965d9f128 2024-11-13T03:51:56,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c6df0e874bf64467b6388c2eaebe1deb 2024-11-13T03:51:56,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/66b244112eda41748b9fddd33f0aff44 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/66b244112eda41748b9fddd33f0aff44 2024-11-13T03:51:56,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bd8cc59fca45459fa05b240b97b0b6c2 2024-11-13T03:51:56,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/788c28bf307a49ffa09c32f835d113f1 2024-11-13T03:51:56,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0d15c7ff65d9487e90da336756355e59 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0d15c7ff65d9487e90da336756355e59 2024-11-13T03:51:56,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/8c141c9d90e14676bbed8a65b3d16592 2024-11-13T03:51:56,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/bdb9c9c562ac4dbe817cf2d148f4d1cd 2024-11-13T03:51:56,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e9a8bbbcc2254199bb03a9b5f1bbb7df 2024-11-13T03:51:56,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/e0a1a6a71ff94ebfbe5b57a8eb097c5a 2024-11-13T03:51:56,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4a84a56d6f7a4e86864c0a8718235c79 2024-11-13T03:51:56,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5d017e9e70fe4644943e62d1bf2e0ced 2024-11-13T03:51:56,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/96ec26104d2e48c4b1ed4100bd11d64c 2024-11-13T03:51:56,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/6521da637bf3475e8291b35199454663 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/6521da637bf3475e8291b35199454663 2024-11-13T03:51:56,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/38998582e4b24c40b1b4de740a467599 2024-11-13T03:51:56,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/187078b026e74d8aa642453ba318c861 2024-11-13T03:51:56,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/021815b66de14cd08414cafd30f13b0c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/021815b66de14cd08414cafd30f13b0c 2024-11-13T03:51:56,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/4ee949b07b3c465e9f08ebb38d04dc88 2024-11-13T03:51:56,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/eeb8a378495e4ce09ad3fab1489e3173 2024-11-13T03:51:56,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/3560f53ad5e743758ae501a5605d0cda to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/3560f53ad5e743758ae501a5605d0cda 2024-11-13T03:51:56,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/9fb9bb5bab6342858449ae1fbfb57090 2024-11-13T03:51:56,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/5c863055423245bf8a281f08e35a2343 2024-11-13T03:51:56,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c900c19f8a7d47258b77e358b2c27dd9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/c900c19f8a7d47258b77e358b2c27dd9 2024-11-13T03:51:56,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0764acb77f9c452a82d78580c5597a3b 2024-11-13T03:51:56,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/b9a9b2b0f9d3429bbe63a5734735b1ea 2024-11-13T03:51:56,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/30b16589e41d4684916cae293dbac9cf 2024-11-13T03:51:56,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/93ead78c443d40bf911c444c063f80ac, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/203d4e2e78544d50946dfce60d456795, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/59316c4e359d48dbaa66e81a57817fd8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/5b43aa5f161b4adcb4a042c5e267892e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/cf45d090e8d745938f92537da4c79bdb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9980bd294896464b96890b977b893f9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/23bad0029fa44b669ff79bedaf169aa9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7] to archive 2024-11-13T03:51:56,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:51:56,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/d24b9233e8df4d5bbc2dcdb1aa5853a7 2024-11-13T03:51:56,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/610b622aa17942beaae6d771c20a4f81 2024-11-13T03:51:56,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/93ead78c443d40bf911c444c063f80ac to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/93ead78c443d40bf911c444c063f80ac 2024-11-13T03:51:56,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8f6c89a45ca342f79cf2bd0d40c6326c 2024-11-13T03:51:56,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/f8ce7230b0d14829b5413448d55ef01f 2024-11-13T03:51:56,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/203d4e2e78544d50946dfce60d456795 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/203d4e2e78544d50946dfce60d456795 2024-11-13T03:51:56,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/51a4040e823748a4903da41e576f65c3 2024-11-13T03:51:56,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6d3fb5e7b5e649d1ba2f73aee6180660 2024-11-13T03:51:56,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/08e63a151053463cb5ee0f5943e49d34 2024-11-13T03:51:56,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/59316c4e359d48dbaa66e81a57817fd8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/59316c4e359d48dbaa66e81a57817fd8 2024-11-13T03:51:56,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9342ba649e814b6e804205218009ee87 2024-11-13T03:51:56,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1663421c238043288f290c68296b87f1 2024-11-13T03:51:56,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2f742f79d115433c824fbc97e05d41a5 2024-11-13T03:51:56,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/5b43aa5f161b4adcb4a042c5e267892e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/5b43aa5f161b4adcb4a042c5e267892e 2024-11-13T03:51:56,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a5aeb87441674a72a6345c661be1c493 2024-11-13T03:51:56,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/e30ab289fddd4205a9f6e211e83be644 2024-11-13T03:51:56,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/cf45d090e8d745938f92537da4c79bdb to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/cf45d090e8d745938f92537da4c79bdb 2024-11-13T03:51:56,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/2c54825baa664f7b927ab316963a943c 2024-11-13T03:51:56,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/8fc016bd8514457780d171fac8bf9c0f 2024-11-13T03:51:56,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9980bd294896464b96890b977b893f9c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9980bd294896464b96890b977b893f9c 2024-11-13T03:51:56,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/700e0813a0904f20872b43b96e44f5b6 2024-11-13T03:51:56,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/c60ffaedd6b0406cb9f89bedf1545d52 2024-11-13T03:51:56,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/23bad0029fa44b669ff79bedaf169aa9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/23bad0029fa44b669ff79bedaf169aa9 2024-11-13T03:51:56,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/1ce579cab9554b78b8eb1881d8009bc9 2024-11-13T03:51:56,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/9534b0f6712a4cfda27edefdde4a8118 2024-11-13T03:51:56,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a40f0a7be2274b13b424b7f1547ed5d7 2024-11-13T03:51:56,736 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/recovered.edits/467.seqid, newMaxSeqId=467, maxSeqId=1 2024-11-13T03:51:56,737 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. 2024-11-13T03:51:56,737 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for e3e30c23eeb6b0b0afd7668c5b9a55d3: Waiting for close lock at 1731469915380Running coprocessor pre-close hooks at 1731469915380Disabling compacts and flushes for region at 1731469915380Disabling writes for close at 1731469915381 (+1 ms)Obtaining lock to block concurrent updates at 1731469915381Preparing flush snapshotting stores in e3e30c23eeb6b0b0afd7668c5b9a55d3 at 1731469915381Finished memstore snapshotting TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731469915382 (+1 ms)Flushing stores of TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3. at 1731469915383 (+1 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/A: creating writer at 1731469915383Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/A: appending metadata at 1731469915394 (+11 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/A: closing flushed file at 1731469915394Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/B: creating writer at 1731469915811 (+417 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/B: appending metadata at 1731469915815 (+4 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/B: closing flushed file at 1731469915815Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/C: creating writer at 1731469916231 (+416 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/C: appending metadata at 1731469916234 (+3 ms)Flushing e3e30c23eeb6b0b0afd7668c5b9a55d3/C: closing flushed file at 1731469916235 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7239878f: reopening flushed file at 1731469916648 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d80785e: reopening flushed file at 1731469916655 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@714ef6e1: reopening flushed file at 1731469916658 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e3e30c23eeb6b0b0afd7668c5b9a55d3 in 1281ms, sequenceid=464, compaction requested=true at 1731469916662 (+4 ms)Writing region close event to WAL at 1731469916733 (+71 ms)Running coprocessor post-close hooks at 1731469916736 (+3 ms)Closed at 1731469916737 (+1 ms) 2024-11-13T03:51:56,739 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:56,739 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=e3e30c23eeb6b0b0afd7668c5b9a55d3, regionState=CLOSED 2024-11-13T03:51:56,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:51:56,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-13T03:51:56,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure e3e30c23eeb6b0b0afd7668c5b9a55d3, server=ac1a51731247,42123,1731469817167 in 1.5140 sec 2024-11-13T03:51:56,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-13T03:51:56,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e3e30c23eeb6b0b0afd7668c5b9a55d3, UNASSIGN in 1.5190 sec 2024-11-13T03:51:56,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T03:51:56,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-13T03:51:56,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469916746"}]},"ts":"1731469916746"} 2024-11-13T03:51:56,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:51:56,747 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:51:56,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-13T03:51:56,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T03:51:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T03:51:57,368 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:57,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:51:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,375 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-13T03:51:57,377 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,382 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:57,385 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/recovered.edits] 2024-11-13T03:51:57,388 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/21da9c2a82d54fd49233f2e78f1492db to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/21da9c2a82d54fd49233f2e78f1492db 2024-11-13T03:51:57,390 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/2e1c59323d504cfdb96a03d4e494a717 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/2e1c59323d504cfdb96a03d4e494a717 2024-11-13T03:51:57,392 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/cee0c32c3fcc4a178fd7a1aa90e34440 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/A/cee0c32c3fcc4a178fd7a1aa90e34440 2024-11-13T03:51:57,395 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0625642f16f6427baeba4d2f29686f86 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/0625642f16f6427baeba4d2f29686f86 2024-11-13T03:51:57,397 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/1a0dadbb2e3747a29e82a5ab7cc094c2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/1a0dadbb2e3747a29e82a5ab7cc094c2 2024-11-13T03:51:57,399 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/87a0553b5d5d4a62a41e582b89d72145 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/B/87a0553b5d5d4a62a41e582b89d72145 2024-11-13T03:51:57,402 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/141d747c8dcb483fad08b84c2eecbece to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/141d747c8dcb483fad08b84c2eecbece 2024-11-13T03:51:57,404 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6da822e0922d463fab6bf4cf50ce39ac to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/6da822e0922d463fab6bf4cf50ce39ac 2024-11-13T03:51:57,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a98bb1ad3f88477ea19a447be72b8ca3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/C/a98bb1ad3f88477ea19a447be72b8ca3 2024-11-13T03:51:57,409 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/recovered.edits/467.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3/recovered.edits/467.seqid 2024-11-13T03:51:57,410 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/e3e30c23eeb6b0b0afd7668c5b9a55d3 2024-11-13T03:51:57,410 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:51:57,412 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,415 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:51:57,417 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:51:57,417 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,417 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:51:57,418 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731469917417"}]},"ts":"9223372036854775807"} 2024-11-13T03:51:57,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:51:57,419 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e3e30c23eeb6b0b0afd7668c5b9a55d3, NAME => 'TestAcidGuarantees,,1731469883884.e3e30c23eeb6b0b0afd7668c5b9a55d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:51:57,419 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:51:57,419 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731469917419"}]},"ts":"9223372036854775807"} 2024-11-13T03:51:57,421 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:51:57,421 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-13T03:51:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-13T03:51:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:51:57,488 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:57,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 251), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=8178 (was 8260) 2024-11-13T03:51:57,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8178 2024-11-13T03:51:57,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:51:57,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:51:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:57,520 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:51:57,520 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:57,520 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-13T03:51:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:57,522 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:51:57,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742161_1337 (size=1061) 2024-11-13T03:51:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:57,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:51:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742162_1338 (size=53) 2024-11-13T03:51:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:58,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:51:58,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 42c4710928cbd1be72d51fa776aaa85f, disabling compactions & flushes 2024-11-13T03:51:58,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. after waiting 0 ms 2024-11-13T03:51:58,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,346 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42c4710928cbd1be72d51fa776aaa85f: Waiting for close lock at 1731469918345Disabling compacts and flushes for region at 1731469918345Disabling writes for close at 1731469918346 (+1 ms)Writing region close event to WAL at 1731469918346Closed at 1731469918346 2024-11-13T03:51:58,349 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:51:58,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469918349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469918349"}]},"ts":"1731469918349"} 2024-11-13T03:51:58,354 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:51:58,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:51:58,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469918355"}]},"ts":"1731469918355"} 2024-11-13T03:51:58,357 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:51:58,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, ASSIGN}] 2024-11-13T03:51:58,359 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, ASSIGN 2024-11-13T03:51:58,360 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:51:58,512 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:58,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, ASSIGN because future has completed 2024-11-13T03:51:58,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:51:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:58,680 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,680 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:51:58,681 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,681 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:51:58,681 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,682 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,684 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,685 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:58,686 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName A 2024-11-13T03:51:58,686 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:58,686 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:58,686 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,688 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:58,688 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName B 2024-11-13T03:51:58,688 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:58,688 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:58,688 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,690 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:51:58,690 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName C 2024-11-13T03:51:58,690 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:51:58,690 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:51:58,691 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,691 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,691 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,692 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,693 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,693 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,693 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:51:58,695 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,697 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:51:58,698 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 42c4710928cbd1be72d51fa776aaa85f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69542226, jitterRate=0.03625991940498352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:51:58,698 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:51:58,699 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 42c4710928cbd1be72d51fa776aaa85f: Running coprocessor pre-open hook at 1731469918682Writing region info on filesystem at 1731469918682Initializing all the Stores at 1731469918683 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469918683Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469918683Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469918683Cleaning up temporary data from old regions at 1731469918693 (+10 ms)Running coprocessor post-open hooks at 1731469918698 (+5 ms)Region opened successfully at 1731469918699 (+1 ms) 2024-11-13T03:51:58,700 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., pid=95, masterSystemTime=1731469918672 2024-11-13T03:51:58,702 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,702 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:51:58,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:51:58,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:51:58,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-13T03:51:58,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 in 188 msec 2024-11-13T03:51:58,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T03:51:58,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, ASSIGN in 351 msec 2024-11-13T03:51:58,713 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:51:58,713 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469918713"}]},"ts":"1731469918713"} 2024-11-13T03:51:58,716 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:51:58,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:51:58,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-11-13T03:51:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T03:51:59,678 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:51:59,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:59,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:51:59,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:51:59,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:51:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:51:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:51:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8bb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:51:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:51:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:51:59,687 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:51:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2550cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:51:59,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:51:59,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:51:59,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:51:59,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:51:59,692 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:51:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:51:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ffd6ca7 2024-11-13T03:51:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:51:59,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:51:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:51:59,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:51:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T03:51:59,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742163_1339 (size=1097) 2024-11-13T03:52:00,113 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T03:52:00,113 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T03:52:00,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:52:00,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE}] 2024-11-13T03:52:00,125 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE 2024-11-13T03:52:00,126 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE because future has completed 2024-11-13T03:52:00,129 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:52:00,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:52:00,283 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,283 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:52:00,283 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 42c4710928cbd1be72d51fa776aaa85f, disabling compactions & flushes 2024-11-13T03:52:00,283 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,283 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,283 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. after waiting 0 ms 2024-11-13T03:52:00,283 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,292 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T03:52:00,293 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,293 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 42c4710928cbd1be72d51fa776aaa85f: Waiting for close lock at 1731469920283Running coprocessor pre-close hooks at 1731469920283Disabling compacts and flushes for region at 1731469920283Disabling writes for close at 1731469920283Writing region close event to WAL at 1731469920284 (+1 ms)Running coprocessor post-close hooks at 1731469920293 (+9 ms)Closed at 1731469920293 2024-11-13T03:52:00,294 WARN [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 42c4710928cbd1be72d51fa776aaa85f to self. 2024-11-13T03:52:00,297 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,298 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=CLOSED 2024-11-13T03:52:00,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:52:00,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-13T03:52:00,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 in 172 msec 2024-11-13T03:52:00,305 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE; state=CLOSED, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=true 2024-11-13T03:52:00,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE because future has completed 2024-11-13T03:52:00,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:52:00,622 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,623 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:52:00,623 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,624 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:52:00,624 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,624 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,627 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,629 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:00,629 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName A 2024-11-13T03:52:00,630 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:00,631 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:00,631 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,632 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:00,632 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName B 2024-11-13T03:52:00,632 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:00,633 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:00,633 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,633 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:00,634 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42c4710928cbd1be72d51fa776aaa85f columnFamilyName C 2024-11-13T03:52:00,634 DEBUG [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:00,634 INFO [StoreOpener-42c4710928cbd1be72d51fa776aaa85f-1 {}] regionserver.HStore(327): Store=42c4710928cbd1be72d51fa776aaa85f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:00,634 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,634 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,635 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,636 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,637 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,637 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,638 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:52:00,639 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,640 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 42c4710928cbd1be72d51fa776aaa85f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65374193, jitterRate=-0.025848612189292908}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:52:00,640 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,641 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 42c4710928cbd1be72d51fa776aaa85f: Running coprocessor pre-open hook at 1731469920624Writing region info on filesystem at 1731469920624Initializing all the Stores at 1731469920626 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469920626Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469920627 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469920627Cleaning up temporary data from old regions at 1731469920637 (+10 ms)Running coprocessor post-open hooks at 1731469920640 (+3 ms)Region opened successfully at 1731469920641 (+1 ms) 2024-11-13T03:52:00,642 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., pid=100, masterSystemTime=1731469920615 2024-11-13T03:52:00,644 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,644 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=OPEN, openSeqNum=5, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:52:00,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-13T03:52:00,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 in 187 msec 2024-11-13T03:52:00,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T03:52:00,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, REOPEN/MOVE in 527 msec 2024-11-13T03:52:00,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-13T03:52:00,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-13T03:52:00,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 959 msec 2024-11-13T03:52:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-13T03:52:00,657 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:00,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700609ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d04d6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754b9916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614ab3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67df0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77024f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434be25f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b724df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36787e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302a9425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676af702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bd4ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,678 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c46e6d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,680 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df2037c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d44774, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,684 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac7373c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:00,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:00,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3b7d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:00,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a7475d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:00,689 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,689 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,690 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,690 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,690 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,691 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:00,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,693 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:00,693 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,694 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:00,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:00,694 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,696 DEBUG [Thread-1553 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,696 DEBUG [Thread-1553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,698 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,700 DEBUG [Thread-1555 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,702 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:00,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:00,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:52:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:00,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111335641fc2cf444220b2286023646de17d_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469920735/Put/seqid=0 2024-11-13T03:52:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742164_1340 (size=12154) 2024-11-13T03:52:00,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:00,790 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111335641fc2cf444220b2286023646de17d_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335641fc2cf444220b2286023646de17d_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:00,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/e6500b4f771c4fcb940649ec86c2b326, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:00,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/e6500b4f771c4fcb940649ec86c2b326 is 175, key is test_row_0/A:col10/1731469920735/Put/seqid=0 2024-11-13T03:52:00,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469980785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742165_1341 (size=30955) 2024-11-13T03:52:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:00,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469980785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469980786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469980787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469980788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:00,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:00,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469980896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469980907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469980907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469980907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469980907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:01,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469981117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469981117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469981117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469981118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469981118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,195 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/e6500b4f771c4fcb940649ec86c2b326 2024-11-13T03:52:01,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/151c34932ae54be79b8cdfea59888539 is 50, key is test_row_0/B:col10/1731469920735/Put/seqid=0 2024-11-13T03:52:01,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742166_1342 (size=12001) 2024-11-13T03:52:01,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469981427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469981427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469981437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469981437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469981437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:01,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/151c34932ae54be79b8cdfea59888539 2024-11-13T03:52:01,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/f1ea4b791cc4442fa835550844be04f3 is 50, key is test_row_0/C:col10/1731469920735/Put/seqid=0 2024-11-13T03:52:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742167_1343 (size=12001) 2024-11-13T03:52:01,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/f1ea4b791cc4442fa835550844be04f3 2024-11-13T03:52:01,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/e6500b4f771c4fcb940649ec86c2b326 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326 2024-11-13T03:52:01,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T03:52:01,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/151c34932ae54be79b8cdfea59888539 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539 2024-11-13T03:52:01,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T03:52:01,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/f1ea4b791cc4442fa835550844be04f3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3 2024-11-13T03:52:01,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T03:52:01,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 42c4710928cbd1be72d51fa776aaa85f in 963ms, sequenceid=16, compaction requested=false 2024-11-13T03:52:01,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:01,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T03:52:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:01,768 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:52:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b164342822774c3f997be71d01e69f6b_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469920782/Put/seqid=0 2024-11-13T03:52:01,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742168_1344 (size=12154) 2024-11-13T03:52:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469981952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469981953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469981954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469981957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469981958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469982067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469982067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469982067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469982067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:02,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b164342822774c3f997be71d01e69f6b_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b164342822774c3f997be71d01e69f6b_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/57ed4780d931443f99c9ed3accc632c9, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/57ed4780d931443f99c9ed3accc632c9 is 175, key is test_row_0/A:col10/1731469920782/Put/seqid=0 2024-11-13T03:52:02,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742169_1345 (size=30955) 2024-11-13T03:52:02,274 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:52:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469982277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469982277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469982277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469982277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469982587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469982596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469982597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469982597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:02,608 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/57ed4780d931443f99c9ed3accc632c9 2024-11-13T03:52:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/7355b3d2b7374edd8e43d9d71e499406 is 50, key is test_row_0/B:col10/1731469920782/Put/seqid=0 2024-11-13T03:52:02,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742170_1346 (size=12001) 2024-11-13T03:52:02,618 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/7355b3d2b7374edd8e43d9d71e499406 2024-11-13T03:52:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/d6260f36b3004d69a03ae3d94204051d is 50, key is test_row_0/C:col10/1731469920782/Put/seqid=0 2024-11-13T03:52:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742171_1347 (size=12001) 2024-11-13T03:52:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469982977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:02,981 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,034 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/d6260f36b3004d69a03ae3d94204051d 2024-11-13T03:52:03,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/57ed4780d931443f99c9ed3accc632c9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9 2024-11-13T03:52:03,042 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9, entries=150, sequenceid=36, filesize=30.2 K 2024-11-13T03:52:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/7355b3d2b7374edd8e43d9d71e499406 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406 2024-11-13T03:52:03,047 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T03:52:03,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/d6260f36b3004d69a03ae3d94204051d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d 2024-11-13T03:52:03,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T03:52:03,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42c4710928cbd1be72d51fa776aaa85f in 1283ms, sequenceid=36, compaction requested=false 2024-11-13T03:52:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T03:52:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T03:52:03,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T03:52:03,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3590 sec 2024-11-13T03:52:03,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3650 sec 2024-11-13T03:52:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:03,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:03,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:03,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111396c5ae0467aa4b779e337b4c6681ed07_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:03,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742172_1348 (size=14594) 2024-11-13T03:52:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469983135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469983138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469983145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469983145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469983256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469983257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469983257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469983257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469983466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469983467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469983467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469983467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:03,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111396c5ae0467aa4b779e337b4c6681ed07_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111396c5ae0467aa4b779e337b4c6681ed07_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:03,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/3305dc1bfaa24d4d84a788d1891092d9, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:03,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/3305dc1bfaa24d4d84a788d1891092d9 is 175, key is test_row_0/A:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:03,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742173_1349 (size=39549) 2024-11-13T03:52:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469983777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469983777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469983777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469983777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:03,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/3305dc1bfaa24d4d84a788d1891092d9 2024-11-13T03:52:03,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/630788315e324b5a8ea57b1c2c7fa901 is 50, key is test_row_0/B:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:03,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742174_1350 (size=12001) 2024-11-13T03:52:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469984286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469984286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469984287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:04,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469984287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:04,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/630788315e324b5a8ea57b1c2c7fa901 2024-11-13T03:52:04,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/017267478fa141f498fa30c86cef331c is 50, key is test_row_0/C:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:04,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742175_1351 (size=12001) 2024-11-13T03:52:04,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/017267478fa141f498fa30c86cef331c 2024-11-13T03:52:04,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/3305dc1bfaa24d4d84a788d1891092d9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9 2024-11-13T03:52:04,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9, entries=200, sequenceid=49, filesize=38.6 K 2024-11-13T03:52:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/630788315e324b5a8ea57b1c2c7fa901 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901 2024-11-13T03:52:04,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T03:52:04,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/017267478fa141f498fa30c86cef331c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c 2024-11-13T03:52:04,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T03:52:04,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 42c4710928cbd1be72d51fa776aaa85f in 1662ms, sequenceid=49, compaction requested=true 2024-11-13T03:52:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:04,762 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:04,762 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:04,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:04,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:04,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:04,764 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:04,765 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:04,765 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=99.1 K 2024-11-13T03:52:04,765 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:04,765 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9] 2024-11-13T03:52:04,765 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.2 K 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6500b4f771c4fcb940649ec86c2b326, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731469920735 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 151c34932ae54be79b8cdfea59888539, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731469920735 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ed4780d931443f99c9ed3accc632c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731469920782 2024-11-13T03:52:04,765 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7355b3d2b7374edd8e43d9d71e499406, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731469920782 2024-11-13T03:52:04,766 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3305dc1bfaa24d4d84a788d1891092d9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921942 2024-11-13T03:52:04,766 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 630788315e324b5a8ea57b1c2c7fa901, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921953 2024-11-13T03:52:04,777 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:04,778 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:04,778 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a842facc29a8426b95f530af39a11bec is 50, key is test_row_0/B:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:04,780 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c5c605337abd4d45ad3924df5cda01a5_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:04,782 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c5c605337abd4d45ad3924df5cda01a5_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:04,782 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5c605337abd4d45ad3924df5cda01a5_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742176_1352 (size=12104) 2024-11-13T03:52:04,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742177_1353 (size=4469) 2024-11-13T03:52:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T03:52:04,867 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:04,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T03:52:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:04,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:04,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:04,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:04,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:52:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:04,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:04,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:04,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:05,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133539737815874fb18e85c5302a098487_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469924992/Put/seqid=0 2024-11-13T03:52:05,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742178_1354 (size=14594) 2024-11-13T03:52:05,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469985048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469985167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:05,195 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:05,195 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/b4ec9728888643a5a9c9313f75f18a87 is 175, key is test_row_0/A:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:05,198 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a842facc29a8426b95f530af39a11bec as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a842facc29a8426b95f530af39a11bec 2024-11-13T03:52:05,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742179_1355 (size=31058) 2024-11-13T03:52:05,203 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into a842facc29a8426b95f530af39a11bec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:05,203 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:05,203 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=13, startTime=1731469924762; duration=0sec 2024-11-13T03:52:05,203 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:05,203 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:05,203 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:05,204 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:05,204 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/b4ec9728888643a5a9c9313f75f18a87 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87 2024-11-13T03:52:05,204 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:05,205 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,205 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.2 K 2024-11-13T03:52:05,205 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ea4b791cc4442fa835550844be04f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731469920735 2024-11-13T03:52:05,205 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d6260f36b3004d69a03ae3d94204051d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731469920782 2024-11-13T03:52:05,205 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 017267478fa141f498fa30c86cef331c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921953 2024-11-13T03:52:05,208 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into b4ec9728888643a5a9c9313f75f18a87(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:05,208 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:05,208 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=13, startTime=1731469924762; duration=0sec 2024-11-13T03:52:05,209 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:05,209 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:05,211 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:05,211 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/84d8fef6543b4e66b671f6d41430a2ee is 50, key is test_row_0/C:col10/1731469921953/Put/seqid=0 2024-11-13T03:52:05,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742180_1356 (size=12104) 2024-11-13T03:52:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469985306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469985307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469985307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,313 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:05,314 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469985307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,314 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,314 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469985386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:05,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133539737815874fb18e85c5302a098487_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133539737815874fb18e85c5302a098487_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:05,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/da1cb272d0374bd78edb31c64c56ac0c, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:05,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/da1cb272d0374bd78edb31c64c56ac0c is 175, key is test_row_0/A:col10/1731469924992/Put/seqid=0 2024-11-13T03:52:05,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742181_1357 (size=39549) 2024-11-13T03:52:05,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:05,620 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/84d8fef6543b4e66b671f6d41430a2ee as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/84d8fef6543b4e66b671f6d41430a2ee 2024-11-13T03:52:05,624 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into 84d8fef6543b4e66b671f6d41430a2ee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:05,624 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:05,624 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=13, startTime=1731469924763; duration=0sec 2024-11-13T03:52:05,624 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:05,624 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:05,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469985707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:05,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/da1cb272d0374bd78edb31c64c56ac0c 2024-11-13T03:52:05,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/43b3885c125d4424a34bd390fce390b2 is 50, key is test_row_0/B:col10/1731469924992/Put/seqid=0 2024-11-13T03:52:05,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742182_1358 (size=12001) 2024-11-13T03:52:05,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:05,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:06,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469986217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:06,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/43b3885c125d4424a34bd390fce390b2 2024-11-13T03:52:06,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/55498bcab279437f893996dd35dc41a3 is 50, key is test_row_0/C:col10/1731469924992/Put/seqid=0 2024-11-13T03:52:06,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742183_1359 (size=12001) 2024-11-13T03:52:06,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/55498bcab279437f893996dd35dc41a3 2024-11-13T03:52:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/da1cb272d0374bd78edb31c64c56ac0c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c 2024-11-13T03:52:06,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c, entries=200, sequenceid=69, filesize=38.6 K 2024-11-13T03:52:06,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/43b3885c125d4424a34bd390fce390b2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2 2024-11-13T03:52:06,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:06,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:06,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:06,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2, entries=150, sequenceid=69, filesize=11.7 K 2024-11-13T03:52:06,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/55498bcab279437f893996dd35dc41a3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3 2024-11-13T03:52:06,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3, entries=150, sequenceid=69, filesize=11.7 K 2024-11-13T03:52:06,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42c4710928cbd1be72d51fa776aaa85f in 1262ms, sequenceid=69, compaction requested=false 2024-11-13T03:52:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:06,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T03:52:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,396 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:52:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:06,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:06,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:06,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343820fc8ada9406e927a501474e4db2c_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469925023/Put/seqid=0 2024-11-13T03:52:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742184_1360 (size=12154) 2024-11-13T03:52:06,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,415 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343820fc8ada9406e927a501474e4db2c_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343820fc8ada9406e927a501474e4db2c_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:06,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6cdfd287b92b4c0eb930c8b070d17ba1, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:06,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6cdfd287b92b4c0eb930c8b070d17ba1 is 175, key is test_row_0/A:col10/1731469925023/Put/seqid=0 2024-11-13T03:52:06,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742185_1361 (size=30955) 2024-11-13T03:52:06,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6cdfd287b92b4c0eb930c8b070d17ba1 2024-11-13T03:52:06,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a3267e30f51d4f2e94d9380dd71db951 is 50, key is test_row_0/B:col10/1731469925023/Put/seqid=0 2024-11-13T03:52:06,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742186_1362 (size=12001) 2024-11-13T03:52:06,437 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a3267e30f51d4f2e94d9380dd71db951 2024-11-13T03:52:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/861dccd6cdfe4a47af0ff985a2d59cb6 is 50, key is test_row_0/C:col10/1731469925023/Put/seqid=0 2024-11-13T03:52:06,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742187_1363 (size=12001) 2024-11-13T03:52:06,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T03:52:06,757 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T03:52:06,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/861dccd6cdfe4a47af0ff985a2d59cb6 2024-11-13T03:52:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6cdfd287b92b4c0eb930c8b070d17ba1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1 2024-11-13T03:52:06,889 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1, entries=150, sequenceid=84, filesize=30.2 K 2024-11-13T03:52:06,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a3267e30f51d4f2e94d9380dd71db951 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951 2024-11-13T03:52:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,893 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T03:52:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/861dccd6cdfe4a47af0ff985a2d59cb6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6 2024-11-13T03:52:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T03:52:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 42c4710928cbd1be72d51fa776aaa85f in 502ms, sequenceid=84, compaction requested=true 2024-11-13T03:52:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T03:52:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T03:52:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T03:52:06,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-11-13T03:52:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0370 sec 2024-11-13T03:52:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T03:52:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T03:52:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:07,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T03:52:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T03:52:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T03:52:07,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-13T03:52:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 160 msec 2024-11-13T03:52:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T03:52:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:07,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:07,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113938c05c2731e411a990557ef93c6f974_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:07,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742188_1364 (size=21918) 2024-11-13T03:52:07,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:07,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113938c05c2731e411a990557ef93c6f974_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113938c05c2731e411a990557ef93c6f974_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:07,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/76c84f99557c44ceb13b946c5283e825, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:07,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/76c84f99557c44ceb13b946c5283e825 is 175, key is test_row_0/A:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:07,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742189_1365 (size=65323) 2024-11-13T03:52:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469987414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469987424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469987425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469987426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469927410, max timestamp=1731469927424Num uniques 859; 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469927410, max timestamp=1731469927424Num uniques 859; 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469927410, max timestamp=1731469927424Num uniques 859; 2024-11-13T03:52:07,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:07,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469987536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469987537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469987537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469987537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469987541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/76c84f99557c44ceb13b946c5283e825 2024-11-13T03:52:07,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/4bb1888903f546d7b8a28a3c8a939ecb is 50, key is test_row_0/B:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742190_1366 (size=12001) 2024-11-13T03:52:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469987877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:07,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:07,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469988067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469988077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469988077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469988077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:08,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:08,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:08,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469988197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/4bb1888903f546d7b8a28a3c8a939ecb 2024-11-13T03:52:08,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/cd7ba49d10c64138bd72cba2956a8307 is 50, key is test_row_0/C:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:08,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742191_1367 (size=12001) 2024-11-13T03:52:08,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/cd7ba49d10c64138bd72cba2956a8307 2024-11-13T03:52:08,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/76c84f99557c44ceb13b946c5283e825 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825 2024-11-13T03:52:08,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825, entries=350, sequenceid=95, filesize=63.8 K 2024-11-13T03:52:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/4bb1888903f546d7b8a28a3c8a939ecb as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb 2024-11-13T03:52:08,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T03:52:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/cd7ba49d10c64138bd72cba2956a8307 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307 2024-11-13T03:52:08,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T03:52:08,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 42c4710928cbd1be72d51fa776aaa85f in 884ms, sequenceid=95, compaction requested=true 2024-11-13T03:52:08,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:08,261 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:08,261 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:08,262 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,262 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=163.0 K 2024-11-13T03:52:08,262 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825] 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4ec9728888643a5a9c9313f75f18a87, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921953 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:08,262 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,262 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a842facc29a8426b95f530af39a11bec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=47.0 K 2024-11-13T03:52:08,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1cb272d0374bd78edb31c64c56ac0c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731469923137 2024-11-13T03:52:08,263 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdfd287b92b4c0eb930c8b070d17ba1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731469925023 2024-11-13T03:52:08,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a842facc29a8426b95f530af39a11bec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921953 2024-11-13T03:52:08,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b3885c125d4424a34bd390fce390b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731469923137 2024-11-13T03:52:08,263 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c84f99557c44ceb13b946c5283e825, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:08,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a3267e30f51d4f2e94d9380dd71db951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731469925023 2024-11-13T03:52:08,264 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb1888903f546d7b8a28a3c8a939ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:08,268 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:08,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T03:52:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,269 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T03:52:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:08,274 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135463214982c44bf1a2b3acf6b629b8fc_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:08,275 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:08,275 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/1694f098e54d4929833c4608b275efa3 is 50, key is test_row_0/B:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:08,277 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135463214982c44bf1a2b3acf6b629b8fc_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:08,277 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135463214982c44bf1a2b3acf6b629b8fc_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113250ef05a4c794fa0b6fc21794e70d2cd_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469927485/Put/seqid=0 2024-11-13T03:52:08,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742192_1368 (size=12241) 2024-11-13T03:52:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742194_1370 (size=12204) 2024-11-13T03:52:08,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742193_1369 (size=4469) 2024-11-13T03:52:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:08,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469988620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469988621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469988621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469988621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:08,687 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/1694f098e54d4929833c4608b275efa3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/1694f098e54d4929833c4608b275efa3 2024-11-13T03:52:08,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113250ef05a4c794fa0b6fc21794e70d2cd_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113250ef05a4c794fa0b6fc21794e70d2cd_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8e550db5d02b40f6be9ce1f2537c3c39, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8e550db5d02b40f6be9ce1f2537c3c39 is 175, key is test_row_0/A:col10/1731469927485/Put/seqid=0 2024-11-13T03:52:08,692 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into 1694f098e54d4929833c4608b275efa3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:08,692 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:08,692 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=12, startTime=1731469928261; duration=0sec 2024-11-13T03:52:08,692 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:08,692 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:08,692 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:08,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:08,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:08,693 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:08,694 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/84d8fef6543b4e66b671f6d41430a2ee, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=47.0 K 2024-11-13T03:52:08,694 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d8fef6543b4e66b671f6d41430a2ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731469921953 2024-11-13T03:52:08,694 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 55498bcab279437f893996dd35dc41a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731469923137 2024-11-13T03:52:08,694 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 861dccd6cdfe4a47af0ff985a2d59cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731469925023 2024-11-13T03:52:08,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742195_1371 (size=31005) 2024-11-13T03:52:08,695 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7ba49d10c64138bd72cba2956a8307, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:08,706 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:08,706 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f84e9207cd40eaae14e764bb4d0453 is 175, key is test_row_0/A:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:08,710 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#318 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:08,710 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/b1007749fc6f438983126b468cb6856c is 50, key is test_row_0/C:col10/1731469927375/Put/seqid=0 2024-11-13T03:52:08,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742196_1372 (size=31195) 2024-11-13T03:52:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742197_1373 (size=12241) 2024-11-13T03:52:08,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469988716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469988747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469988747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469988747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469988747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,095 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8e550db5d02b40f6be9ce1f2537c3c39 2024-11-13T03:52:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/50be874de8914ff197538e1ce6ce0ea3 is 50, key is test_row_0/B:col10/1731469927485/Put/seqid=0 2024-11-13T03:52:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742198_1374 (size=12051) 2024-11-13T03:52:09,116 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f84e9207cd40eaae14e764bb4d0453 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453 2024-11-13T03:52:09,121 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/b1007749fc6f438983126b468cb6856c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/b1007749fc6f438983126b468cb6856c 2024-11-13T03:52:09,123 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into d7f84e9207cd40eaae14e764bb4d0453(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:09,123 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:09,123 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=12, startTime=1731469928261; duration=0sec 2024-11-13T03:52:09,123 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:09,123 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:09,125 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into b1007749fc6f438983126b468cb6856c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:09,125 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:09,125 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=12, startTime=1731469928261; duration=0sec 2024-11-13T03:52:09,125 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:09,125 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:09,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469989286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469989287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469989287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469989287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/50be874de8914ff197538e1ce6ce0ea3 2024-11-13T03:52:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/a0841f6317944df6ac5f276c845244b8 is 50, key is test_row_0/C:col10/1731469927485/Put/seqid=0 2024-11-13T03:52:09,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742199_1375 (size=12051) 2024-11-13T03:52:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469989727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,736 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469989807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469989807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469989807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469989807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:09,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/a0841f6317944df6ac5f276c845244b8 2024-11-13T03:52:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8e550db5d02b40f6be9ce1f2537c3c39 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39 2024-11-13T03:52:09,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39, entries=150, sequenceid=130, filesize=30.3 K 2024-11-13T03:52:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/50be874de8914ff197538e1ce6ce0ea3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3 2024-11-13T03:52:09,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T03:52:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/a0841f6317944df6ac5f276c845244b8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8 2024-11-13T03:52:09,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T03:52:09,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 42c4710928cbd1be72d51fa776aaa85f in 1663ms, sequenceid=130, compaction requested=false 2024-11-13T03:52:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T03:52:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T03:52:09,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T03:52:09,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5820 sec 2024-11-13T03:52:09,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.5870 sec 2024-11-13T03:52:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:10,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:10,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130788c909ecf949bd83a91b7c3976b1d1_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:10,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742200_1376 (size=14794) 2024-11-13T03:52:10,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:10,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130788c909ecf949bd83a91b7c3976b1d1_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130788c909ecf949bd83a91b7c3976b1d1_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:10,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/2ffc079fdc4a467ea408825a2c07d5d1, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:10,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/2ffc079fdc4a467ea408825a2c07d5d1 is 175, key is test_row_0/A:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:10,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742201_1377 (size=39749) 2024-11-13T03:52:10,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469990868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469990869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469930859, max timestamp=1731469930871Num uniques 859; 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469930859, max timestamp=1731469930871Num uniques 859; 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469930859, max timestamp=1731469930871Num uniques 859; 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:10,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469990938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469990940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:10,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469990977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469990977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469991056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469991056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469991197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469991197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/2ffc079fdc4a467ea408825a2c07d5d1 2024-11-13T03:52:11,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/8437c653c9bf48648d5d893a1adf17ed is 50, key is test_row_0/B:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:11,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742202_1378 (size=12151) 2024-11-13T03:52:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469991266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469991267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T03:52:11,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:11,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T03:52:11,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469991507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469991507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:11,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469991576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469991576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:11,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/8437c653c9bf48648d5d893a1adf17ed 2024-11-13T03:52:11,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7ed9e48e13fc4e70b038fcb6a8de3b53 is 50, key is test_row_0/C:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:11,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742203_1379 (size=12151) 2024-11-13T03:52:11,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469991747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:11,758 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:11,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:11,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:11,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469992017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469992017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7ed9e48e13fc4e70b038fcb6a8de3b53 2024-11-13T03:52:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/2ffc079fdc4a467ea408825a2c07d5d1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1 2024-11-13T03:52:12,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1, entries=200, sequenceid=145, filesize=38.8 K 2024-11-13T03:52:12,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/8437c653c9bf48648d5d893a1adf17ed as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed 2024-11-13T03:52:12,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T03:52:12,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7ed9e48e13fc4e70b038fcb6a8de3b53 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53 2024-11-13T03:52:12,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T03:52:12,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 42c4710928cbd1be72d51fa776aaa85f in 1258ms, sequenceid=145, compaction requested=true 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:12,089 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:12,089 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:12,089 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:12,089 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:12,090 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,090 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,090 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=99.6 K 2024-11-13T03:52:12,090 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/1694f098e54d4929833c4608b275efa3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.6 K 2024-11-13T03:52:12,090 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1] 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1694f098e54d4929833c4608b275efa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7f84e9207cd40eaae14e764bb4d0453, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 50be874de8914ff197538e1ce6ce0ea3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731469927485 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e550db5d02b40f6be9ce1f2537c3c39, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731469927485 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8437c653c9bf48648d5d893a1adf17ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:12,090 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ffc079fdc4a467ea408825a2c07d5d1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:12,094 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:12,095 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:12,096 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/d9a3789a597a463e8d0daa5d3790d0c0 is 50, key is test_row_0/B:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:12,097 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113b2077a50752f480f9f355089c1b1a25b_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:12,099 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113b2077a50752f480f9f355089c1b1a25b_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:12,099 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b2077a50752f480f9f355089c1b1a25b_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:12,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:12,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742205_1381 (size=4469) 2024-11-13T03:52:12,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130fcaf0409f5b44d0a2dd390bd4878e35_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469932099/Put/seqid=0 2024-11-13T03:52:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742204_1380 (size=12493) 2024-11-13T03:52:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742206_1382 (size=14794) 2024-11-13T03:52:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:12,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469992138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469992144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469992256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469992257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469992476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469992476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,505 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:12,505 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/39c10fd8df7a4feda6cf4ea76763c0af is 175, key is test_row_0/A:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742207_1383 (size=31447) 2024-11-13T03:52:12,513 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/d9a3789a597a463e8d0daa5d3790d0c0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/d9a3789a597a463e8d0daa5d3790d0c0 2024-11-13T03:52:12,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:12,517 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into d9a3789a597a463e8d0daa5d3790d0c0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:12,518 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:12,518 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=13, startTime=1731469932089; duration=0sec 2024-11-13T03:52:12,518 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:12,518 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:12,518 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:12,518 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:12,519 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:12,519 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,519 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/b1007749fc6f438983126b468cb6856c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.6 K 2024-11-13T03:52:12,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130fcaf0409f5b44d0a2dd390bd4878e35_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130fcaf0409f5b44d0a2dd390bd4878e35_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:12,519 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b1007749fc6f438983126b468cb6856c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731469927293 2024-11-13T03:52:12,519 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a0841f6317944df6ac5f276c845244b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731469927485 2024-11-13T03:52:12,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d30d2303571948e0a4471db26c22b5b8, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:12,519 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed9e48e13fc4e70b038fcb6a8de3b53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:12,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d30d2303571948e0a4471db26c22b5b8 is 175, key is test_row_0/A:col10/1731469932099/Put/seqid=0 2024-11-13T03:52:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742208_1384 (size=39749) 2024-11-13T03:52:12,525 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d30d2303571948e0a4471db26c22b5b8 2024-11-13T03:52:12,528 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:12,529 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/bbd8d2c5d8cb4b9eb508e70be2d1db89 is 50, key is test_row_0/C:col10/1731469928602/Put/seqid=0 2024-11-13T03:52:12,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a73f721151034a3eacb149e7dac42170 is 50, key is test_row_0/B:col10/1731469932099/Put/seqid=0 2024-11-13T03:52:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742209_1385 (size=12493) 2024-11-13T03:52:12,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742210_1386 (size=12151) 2024-11-13T03:52:12,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:12,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469992797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469992797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:12,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:12,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:12,912 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/39c10fd8df7a4feda6cf4ea76763c0af as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af 2024-11-13T03:52:12,916 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into 39c10fd8df7a4feda6cf4ea76763c0af(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:12,916 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:12,916 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=13, startTime=1731469932089; duration=0sec 2024-11-13T03:52:12,916 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:12,916 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:12,939 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/bbd8d2c5d8cb4b9eb508e70be2d1db89 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/bbd8d2c5d8cb4b9eb508e70be2d1db89 2024-11-13T03:52:12,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a73f721151034a3eacb149e7dac42170 2024-11-13T03:52:12,943 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into bbd8d2c5d8cb4b9eb508e70be2d1db89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:12,943 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:12,944 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=13, startTime=1731469932089; duration=0sec 2024-11-13T03:52:12,944 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:12,944 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:12,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/39d05e0b99514091adb008383c5ba61f is 50, key is test_row_0/C:col10/1731469932099/Put/seqid=0 2024-11-13T03:52:12,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742211_1387 (size=12151) 2024-11-13T03:52:12,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/39d05e0b99514091adb008383c5ba61f 2024-11-13T03:52:12,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d30d2303571948e0a4471db26c22b5b8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8 2024-11-13T03:52:12,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8, entries=200, sequenceid=180, filesize=38.8 K 2024-11-13T03:52:12,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/a73f721151034a3eacb149e7dac42170 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170 2024-11-13T03:52:12,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T03:52:12,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/39d05e0b99514091adb008383c5ba61f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f 2024-11-13T03:52:12,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T03:52:12,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 42c4710928cbd1be72d51fa776aaa85f in 869ms, sequenceid=180, compaction requested=false 2024-11-13T03:52:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:13,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:52:13,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:13,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:13,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:13,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:13,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:13,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:13,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de89035291874995b417f2c159cf7f75_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:13,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742212_1388 (size=14794) 2024-11-13T03:52:13,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:13,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de89035291874995b417f2c159cf7f75_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de89035291874995b417f2c159cf7f75_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:13,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c04a303a89a94bfebb6e8f96d39d9c67, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:13,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c04a303a89a94bfebb6e8f96d39d9c67 is 175, key is test_row_0/A:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742213_1389 (size=39749) 2024-11-13T03:52:13,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469993079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469993082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469993196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469993196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469993316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469993317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469993406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469993416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,456 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c04a303a89a94bfebb6e8f96d39d9c67 2024-11-13T03:52:13,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/40b0b939a538478f9b13d9a2a2fa5824 is 50, key is test_row_0/B:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:13,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742214_1390 (size=12151) 2024-11-13T03:52:13,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:13,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469993717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469993727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:13,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/40b0b939a538478f9b13d9a2a2fa5824 2024-11-13T03:52:13,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/43d81afa1fcf444ba4a5885e82d36dfd is 50, key is test_row_0/C:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:13,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742215_1391 (size=12151) 2024-11-13T03:52:13,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:13,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469994237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469994247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/43d81afa1fcf444ba4a5885e82d36dfd 2024-11-13T03:52:14,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c04a303a89a94bfebb6e8f96d39d9c67 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67 2024-11-13T03:52:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67, entries=200, sequenceid=195, filesize=38.8 K 2024-11-13T03:52:14,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/40b0b939a538478f9b13d9a2a2fa5824 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824 2024-11-13T03:52:14,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T03:52:14,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/43d81afa1fcf444ba4a5885e82d36dfd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd 2024-11-13T03:52:14,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T03:52:14,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 42c4710928cbd1be72d51fa776aaa85f in 1259ms, sequenceid=195, compaction requested=true 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:14,289 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T03:52:14,289 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:14,290 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,290 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,290 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=108.3 K 2024-11-13T03:52:14,290 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/d9a3789a597a463e8d0daa5d3790d0c0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.9 K 2024-11-13T03:52:14,290 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67] 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a3789a597a463e8d0daa5d3790d0c0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39c10fd8df7a4feda6cf4ea76763c0af, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a73f721151034a3eacb149e7dac42170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731469930895 2024-11-13T03:52:14,290 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30d2303571948e0a4471db26c22b5b8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731469930895 2024-11-13T03:52:14,291 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b0b939a538478f9b13d9a2a2fa5824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:14,291 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c04a303a89a94bfebb6e8f96d39d9c67, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:14,295 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:14,296 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:14,296 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113978870e54a824afab7d0cfaae17eaf23_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:14,297 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/64b55e9a784c4693b1ff346ce57a4482 is 50, key is test_row_0/B:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:14,299 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113978870e54a824afab7d0cfaae17eaf23_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:14,299 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113978870e54a824afab7d0cfaae17eaf23_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742216_1392 (size=12595) 2024-11-13T03:52:14,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742217_1393 (size=4469) 2024-11-13T03:52:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:14,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:14,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134b35b041e37148fa90fa362d6914019e_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469933068/Put/seqid=0 2024-11-13T03:52:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742218_1394 (size=14794) 2024-11-13T03:52:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469994356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469994362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469994466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469994477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469994686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469994686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:14,705 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/64b55e9a784c4693b1ff346ce57a4482 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/64b55e9a784c4693b1ff346ce57a4482 2024-11-13T03:52:14,709 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into 64b55e9a784c4693b1ff346ce57a4482(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:14,709 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:14,709 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=13, startTime=1731469934289; duration=0sec 2024-11-13T03:52:14,709 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:14,709 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:14,709 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:14,710 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:14,710 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:14,710 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,710 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/bbd8d2c5d8cb4b9eb508e70be2d1db89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=35.9 K 2024-11-13T03:52:14,710 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd8d2c5d8cb4b9eb508e70be2d1db89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731469928602 2024-11-13T03:52:14,710 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d05e0b99514091adb008383c5ba61f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731469930895 2024-11-13T03:52:14,711 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d81afa1fcf444ba4a5885e82d36dfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:14,713 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:14,713 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/806a8c8b948d4778b2fc52da1f1007f9 is 175, key is test_row_0/A:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:14,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742219_1395 (size=31549) 2024-11-13T03:52:14,719 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:14,719 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/042ae51bd45c4a888904a21985a5c586 is 50, key is test_row_0/C:col10/1731469933028/Put/seqid=0 2024-11-13T03:52:14,722 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/806a8c8b948d4778b2fc52da1f1007f9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9 2024-11-13T03:52:14,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742220_1396 (size=12595) 2024-11-13T03:52:14,727 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into 806a8c8b948d4778b2fc52da1f1007f9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:14,727 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:14,727 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=13, startTime=1731469934289; duration=0sec 2024-11-13T03:52:14,727 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:14,727 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:14,731 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/042ae51bd45c4a888904a21985a5c586 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/042ae51bd45c4a888904a21985a5c586 2024-11-13T03:52:14,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,735 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into 042ae51bd45c4a888904a21985a5c586(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:14,735 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,735 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=13, startTime=1731469934289; duration=0sec 2024-11-13T03:52:14,735 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:14,735 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:14,750 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134b35b041e37148fa90fa362d6914019e_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b35b041e37148fa90fa362d6914019e_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:14,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/ace587e38bae44119735b1f1974f6200, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:14,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/ace587e38bae44119735b1f1974f6200 is 175, key is test_row_0/A:col10/1731469933068/Put/seqid=0 2024-11-13T03:52:14,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742221_1397 (size=39749) 2024-11-13T03:52:14,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:14,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469994997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469994997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/ace587e38bae44119735b1f1974f6200 2024-11-13T03:52:15,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/33be778cd58c489fbb8dce62adc665ec is 50, key is test_row_0/B:col10/1731469933068/Put/seqid=0 2024-11-13T03:52:15,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742222_1398 (size=12151) 2024-11-13T03:52:15,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/33be778cd58c489fbb8dce62adc665ec 2024-11-13T03:52:15,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/dd1ffbaf6b4149c1ae0078de5064a925 is 50, key is test_row_0/C:col10/1731469933068/Put/seqid=0 2024-11-13T03:52:15,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742223_1399 (size=12151) 2024-11-13T03:52:15,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469995247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,254 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469995257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,264 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:15,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:15,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469995517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469995517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:15,562 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:52:15,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/dd1ffbaf6b4149c1ae0078de5064a925 2024-11-13T03:52:15,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/ace587e38bae44119735b1f1974f6200 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200 2024-11-13T03:52:15,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200, entries=200, sequenceid=216, filesize=38.8 K 2024-11-13T03:52:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/33be778cd58c489fbb8dce62adc665ec as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec 2024-11-13T03:52:15,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T03:52:15,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/dd1ffbaf6b4149c1ae0078de5064a925 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925 2024-11-13T03:52:15,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T03:52:15,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 42c4710928cbd1be72d51fa776aaa85f in 1259ms, sequenceid=216, compaction requested=false 2024-11-13T03:52:15,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:15,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T03:52:15,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:15,651 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c476569d90414a79a52bcde30a6d9e26_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469934355/Put/seqid=0 2024-11-13T03:52:15,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742224_1400 (size=12304) 2024-11-13T03:52:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469935820, max timestamp=1731469935893Num uniques 859; 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469935820, max timestamp=1731469935893Num uniques 859; 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469935820, max timestamp=1731469935893Num uniques 859; 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:15,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:15,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469995990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:16,068 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c476569d90414a79a52bcde30a6d9e26_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c476569d90414a79a52bcde30a6d9e26_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:16,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/417c8d2328bc402dbec388f505fa7bdc, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/417c8d2328bc402dbec388f505fa7bdc is 175, key is test_row_0/A:col10/1731469934355/Put/seqid=0 2024-11-13T03:52:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742225_1401 (size=31105) 2024-11-13T03:52:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469996107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469996327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,472 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/417c8d2328bc402dbec388f505fa7bdc 2024-11-13T03:52:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/f06f0fd60e6d41fcadd80b12a48209c7 is 50, key is test_row_0/B:col10/1731469934355/Put/seqid=0 2024-11-13T03:52:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742226_1402 (size=12151) 2024-11-13T03:52:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469996537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469996537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:16,546 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:16,546 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469996637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:16,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/f06f0fd60e6d41fcadd80b12a48209c7 2024-11-13T03:52:16,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/da59fbd76e7643a184006ebd5718812c is 50, key is test_row_0/C:col10/1731469934355/Put/seqid=0 2024-11-13T03:52:16,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742227_1403 (size=12151) 2024-11-13T03:52:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469997156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731469997266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731469997266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:17,276 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:17,276 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:17,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/da59fbd76e7643a184006ebd5718812c 2024-11-13T03:52:17,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/417c8d2328bc402dbec388f505fa7bdc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc 2024-11-13T03:52:17,336 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc, entries=150, sequenceid=230, filesize=30.4 K 2024-11-13T03:52:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/f06f0fd60e6d41fcadd80b12a48209c7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7 2024-11-13T03:52:17,339 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T03:52:17,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/da59fbd76e7643a184006ebd5718812c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c 2024-11-13T03:52:17,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T03:52:17,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 42c4710928cbd1be72d51fa776aaa85f in 1694ms, sequenceid=230, compaction requested=true 2024-11-13T03:52:17,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:17,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:17,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T03:52:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T03:52:17,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T03:52:17,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.8360 sec 2024-11-13T03:52:17,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 5.8410 sec 2024-11-13T03:52:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:18,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:18,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113701294265fa944a89ba4fa4dd8e52240_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:18,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742228_1404 (size=14944) 2024-11-13T03:52:18,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:18,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113701294265fa944a89ba4fa4dd8e52240_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113701294265fa944a89ba4fa4dd8e52240_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:18,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8df8fab096ca4bdea551b72af4561cff, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:18,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8df8fab096ca4bdea551b72af4561cff is 175, key is test_row_0/A:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742229_1405 (size=39899) 2024-11-13T03:52:18,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469998243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469998357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33574 deadline: 1731469998557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,564 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33604 deadline: 1731469998567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469998567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:18,576 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:18,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8df8fab096ca4bdea551b72af4561cff 2024-11-13T03:52:18,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/3b6429a1e81d4978956b14dd01ac2e8b is 50, key is test_row_0/B:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742230_1406 (size=12251) 2024-11-13T03:52:18,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469998886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:19,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/3b6429a1e81d4978956b14dd01ac2e8b 2024-11-13T03:52:19,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/9176feee7bdc47a38763adfaf9a525c6 is 50, key is test_row_0/C:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:19,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742231_1407 (size=12251) 2024-11-13T03:52:19,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67] to archive 2024-11-13T03:52:19,387 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:19,389 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/e6500b4f771c4fcb940649ec86c2b326 2024-11-13T03:52:19,389 WARN [regionserver/ac1a51731247:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-13T03:52:19,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/57ed4780d931443f99c9ed3accc632c9 2024-11-13T03:52:19,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/3305dc1bfaa24d4d84a788d1891092d9 2024-11-13T03:52:19,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/b4ec9728888643a5a9c9313f75f18a87 2024-11-13T03:52:19,392 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/da1cb272d0374bd78edb31c64c56ac0c 2024-11-13T03:52:19,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6cdfd287b92b4c0eb930c8b070d17ba1 2024-11-13T03:52:19,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/76c84f99557c44ceb13b946c5283e825 2024-11-13T03:52:19,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f84e9207cd40eaae14e764bb4d0453 2024-11-13T03:52:19,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8e550db5d02b40f6be9ce1f2537c3c39 2024-11-13T03:52:19,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/2ffc079fdc4a467ea408825a2c07d5d1 2024-11-13T03:52:19,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/39c10fd8df7a4feda6cf4ea76763c0af 2024-11-13T03:52:19,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d30d2303571948e0a4471db26c22b5b8 2024-11-13T03:52:19,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c04a303a89a94bfebb6e8f96d39d9c67 2024-11-13T03:52:19,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a842facc29a8426b95f530af39a11bec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/1694f098e54d4929833c4608b275efa3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/d9a3789a597a463e8d0daa5d3790d0c0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824] to archive 2024-11-13T03:52:19,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:19,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/151c34932ae54be79b8cdfea59888539 2024-11-13T03:52:19,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/7355b3d2b7374edd8e43d9d71e499406 2024-11-13T03:52:19,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a842facc29a8426b95f530af39a11bec to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a842facc29a8426b95f530af39a11bec 2024-11-13T03:52:19,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/630788315e324b5a8ea57b1c2c7fa901 2024-11-13T03:52:19,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/43b3885c125d4424a34bd390fce390b2 2024-11-13T03:52:19,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a3267e30f51d4f2e94d9380dd71db951 2024-11-13T03:52:19,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/1694f098e54d4929833c4608b275efa3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/1694f098e54d4929833c4608b275efa3 2024-11-13T03:52:19,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/4bb1888903f546d7b8a28a3c8a939ecb 2024-11-13T03:52:19,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/50be874de8914ff197538e1ce6ce0ea3 2024-11-13T03:52:19,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/d9a3789a597a463e8d0daa5d3790d0c0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/d9a3789a597a463e8d0daa5d3790d0c0 2024-11-13T03:52:19,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/8437c653c9bf48648d5d893a1adf17ed 2024-11-13T03:52:19,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/a73f721151034a3eacb149e7dac42170 2024-11-13T03:52:19,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/40b0b939a538478f9b13d9a2a2fa5824 2024-11-13T03:52:19,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/84d8fef6543b4e66b671f6d41430a2ee, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/b1007749fc6f438983126b468cb6856c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/bbd8d2c5d8cb4b9eb508e70be2d1db89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd] to archive 2024-11-13T03:52:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731469999407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:19,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:19,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/f1ea4b791cc4442fa835550844be04f3 2024-11-13T03:52:19,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d6260f36b3004d69a03ae3d94204051d 2024-11-13T03:52:19,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/84d8fef6543b4e66b671f6d41430a2ee to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/84d8fef6543b4e66b671f6d41430a2ee 2024-11-13T03:52:19,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/017267478fa141f498fa30c86cef331c 2024-11-13T03:52:19,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/55498bcab279437f893996dd35dc41a3 2024-11-13T03:52:19,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/861dccd6cdfe4a47af0ff985a2d59cb6 2024-11-13T03:52:19,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/b1007749fc6f438983126b468cb6856c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/b1007749fc6f438983126b468cb6856c 2024-11-13T03:52:19,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/cd7ba49d10c64138bd72cba2956a8307 2024-11-13T03:52:19,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/a0841f6317944df6ac5f276c845244b8 2024-11-13T03:52:19,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/bbd8d2c5d8cb4b9eb508e70be2d1db89 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/bbd8d2c5d8cb4b9eb508e70be2d1db89 2024-11-13T03:52:19,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7ed9e48e13fc4e70b038fcb6a8de3b53 2024-11-13T03:52:19,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/39d05e0b99514091adb008383c5ba61f 2024-11-13T03:52:19,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ac1a51731247:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/43d81afa1fcf444ba4a5885e82d36dfd 2024-11-13T03:52:19,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/9176feee7bdc47a38763adfaf9a525c6 2024-11-13T03:52:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/8df8fab096ca4bdea551b72af4561cff as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff 2024-11-13T03:52:19,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff, entries=200, sequenceid=267, filesize=39.0 K 2024-11-13T03:52:19,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/3b6429a1e81d4978956b14dd01ac2e8b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b 2024-11-13T03:52:19,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T03:52:19,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/9176feee7bdc47a38763adfaf9a525c6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6 2024-11-13T03:52:19,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T03:52:19,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 42c4710928cbd1be72d51fa776aaa85f in 1268ms, sequenceid=267, compaction requested=true 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:19,450 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:19,450 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:19,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:19,451 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:19,451 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:19,451 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:19,451 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:19,451 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:19,451 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:19,451 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/64b55e9a784c4693b1ff346ce57a4482, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=48.0 K 2024-11-13T03:52:19,451 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=139.0 K 2024-11-13T03:52:19,451 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff] 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b55e9a784c4693b1ff346ce57a4482, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806a8c8b948d4778b2fc52da1f1007f9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 33be778cd58c489fbb8dce62adc665ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731469933068 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace587e38bae44119735b1f1974f6200, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731469933068 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting f06f0fd60e6d41fcadd80b12a48209c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469934349 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417c8d2328bc402dbec388f505fa7bdc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469934349 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6429a1e81d4978956b14dd01ac2e8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:19,452 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8df8fab096ca4bdea551b72af4561cff, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:19,457 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:19,461 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#346 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:19,461 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/b97cd4f810d647079d52ea97ea7be8e8 is 50, key is test_row_0/B:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:19,463 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411137981553702404618b6e0577a407b3b4d_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:19,466 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411137981553702404618b6e0577a407b3b4d_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:19,466 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137981553702404618b6e0577a407b3b4d_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742232_1408 (size=12389) 2024-11-13T03:52:19,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742233_1409 (size=4469) 2024-11-13T03:52:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T03:52:19,707 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:19,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T03:52:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:19,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:19,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:19,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:19,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T03:52:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113802c86a5fd0047049d5f699fb770a05d_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469938241/Put/seqid=0 2024-11-13T03:52:19,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742234_1410 (size=12454) 2024-11-13T03:52:19,876 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/b97cd4f810d647079d52ea97ea7be8e8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/b97cd4f810d647079d52ea97ea7be8e8 2024-11-13T03:52:19,880 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into b97cd4f810d647079d52ea97ea7be8e8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:19,880 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:19,880 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=12, startTime=1731469939450; duration=0sec 2024-11-13T03:52:19,880 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:19,880 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:19,880 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:19,881 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:19,881 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c7d2c08438d346f58661248849fa6793 is 175, key is test_row_0/A:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:19,881 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:19,881 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:19,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:19,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/042ae51bd45c4a888904a21985a5c586, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=48.0 K 2024-11-13T03:52:19,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 042ae51bd45c4a888904a21985a5c586, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731469932111 2024-11-13T03:52:19,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting dd1ffbaf6b4149c1ae0078de5064a925, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731469933068 2024-11-13T03:52:19,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting da59fbd76e7643a184006ebd5718812c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469934349 2024-11-13T03:52:19,883 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9176feee7bdc47a38763adfaf9a525c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:19,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742235_1411 (size=31343) 2024-11-13T03:52:19,893 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:19,893 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/32f8ecb669b7448f842ba18f7b30f53f is 50, key is test_row_0/C:col10/1731469935906/Put/seqid=0 2024-11-13T03:52:19,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742236_1412 (size=12389) 2024-11-13T03:52:19,904 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/32f8ecb669b7448f842ba18f7b30f53f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/32f8ecb669b7448f842ba18f7b30f53f 2024-11-13T03:52:19,907 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into 32f8ecb669b7448f842ba18f7b30f53f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:19,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:19,907 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=12, startTime=1731469939450; duration=0sec 2024-11-13T03:52:19,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:19,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:20,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:20,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113802c86a5fd0047049d5f699fb770a05d_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113802c86a5fd0047049d5f699fb770a05d_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:20,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f5c786dcd143419f3328e6489377de, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:20,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f5c786dcd143419f3328e6489377de is 175, key is test_row_0/A:col10/1731469938241/Put/seqid=0 2024-11-13T03:52:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742237_1413 (size=31255) 2024-11-13T03:52:20,291 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/c7d2c08438d346f58661248849fa6793 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793 2024-11-13T03:52:20,295 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into c7d2c08438d346f58661248849fa6793(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:20,295 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:20,295 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=12, startTime=1731469939450; duration=0sec 2024-11-13T03:52:20,295 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:20,295 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:20,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. as already flushing 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469940490, max timestamp=1731469940520Num uniques 859; 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469940490, max timestamp=1731469940520Num uniques 859; 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469940490, max timestamp=1731469940520Num uniques 859; 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731470000630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:20,685 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f5c786dcd143419f3328e6489377de 2024-11-13T03:52:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/42317b1ab2fc4c90808c0eccf6d5b148 is 50, key is test_row_0/B:col10/1731469938241/Put/seqid=0 2024-11-13T03:52:20,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742238_1414 (size=12301) 2024-11-13T03:52:20,704 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-13T03:52:20,704 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:20,705 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,705 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:20,708 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-13T03:52:20,708 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:20,708 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,708 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:20,712 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-13T03:52:20,712 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:20,712 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,712 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:20,715 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-13T03:52:20,715 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:20,715 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,715 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:20,719 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-13T03:52:20,719 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:20,719 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,719 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:20,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:20,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731470000737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:20,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731470000948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:21,096 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/42317b1ab2fc4c90808c0eccf6d5b148 2024-11-13T03:52:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/1e959a0281684a6cb739b94c42917f92 is 50, key is test_row_0/C:col10/1731469938241/Put/seqid=0 2024-11-13T03:52:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742239_1415 (size=12301) 2024-11-13T03:52:21,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33586 deadline: 1731470001257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33588 deadline: 1731470001296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33560 deadline: 1731470001297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:21,297 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:21,297 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 42c4710928cbd1be72d51fa776aaa85f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:52:21,515 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/1e959a0281684a6cb739b94c42917f92 2024-11-13T03:52:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/d7f5c786dcd143419f3328e6489377de as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de 2024-11-13T03:52:21,532 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de, entries=150, sequenceid=277, filesize=30.5 K 2024-11-13T03:52:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/42317b1ab2fc4c90808c0eccf6d5b148 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148 2024-11-13T03:52:21,537 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T03:52:21,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/1e959a0281684a6cb739b94c42917f92 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92 2024-11-13T03:52:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T03:52:21,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 42c4710928cbd1be72d51fa776aaa85f in 1681ms, sequenceid=277, compaction requested=false 2024-11-13T03:52:21,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:21,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:21,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T03:52:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T03:52:21,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T03:52:21,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-13T03:52:21,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8400 sec 2024-11-13T03:52:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:21,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T03:52:21,771 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-13T03:52:21,771 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:21,771 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:21,772 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:21,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:21,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:21,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131457a20041ae4b66805c49b1fab6aa6c_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:21,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742240_1416 (size=12454) 2024-11-13T03:52:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T03:52:21,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:22,187 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:22,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131457a20041ae4b66805c49b1fab6aa6c_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131457a20041ae4b66805c49b1fab6aa6c_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:22,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/9610d17cc80f41118cd7ca493f57ec49, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:22,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/9610d17cc80f41118cd7ca493f57ec49 is 175, key is test_row_0/A:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:22,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742241_1417 (size=31255) 2024-11-13T03:52:22,591 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-13T03:52:22,591 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:22,591 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:22,592 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:22,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:22,601 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-13T03:52:22,601 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:22,601 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:22,602 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:22,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:22,622 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/9610d17cc80f41118cd7ca493f57ec49 2024-11-13T03:52:22,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/cd3b713c56724233b477ee32dddee539 is 50, key is test_row_0/B:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742242_1418 (size=12301) 2024-11-13T03:52:23,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/cd3b713c56724233b477ee32dddee539 2024-11-13T03:52:23,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7c9341d8cb464a49bb0d8881fc24ce54 is 50, key is test_row_0/C:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742243_1419 (size=12301) 2024-11-13T03:52:23,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7c9341d8cb464a49bb0d8881fc24ce54 2024-11-13T03:52:23,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/9610d17cc80f41118cd7ca493f57ec49 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49 2024-11-13T03:52:23,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49, entries=150, sequenceid=317, filesize=30.5 K 2024-11-13T03:52:23,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/cd3b713c56724233b477ee32dddee539 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539 2024-11-13T03:52:23,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T03:52:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/7c9341d8cb464a49bb0d8881fc24ce54 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54 2024-11-13T03:52:23,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T03:52:23,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=13.42 KB/13740 for 42c4710928cbd1be72d51fa776aaa85f in 1712ms, sequenceid=317, compaction requested=true 2024-11-13T03:52:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42c4710928cbd1be72d51fa776aaa85f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93853 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/A is initiating minor compaction (all files) 2024-11-13T03:52:23,483 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/B is initiating minor compaction (all files) 2024-11-13T03:52:23,483 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/A in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:23,484 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/B in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:23,484 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=91.7 K 2024-11-13T03:52:23,484 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/b97cd4f810d647079d52ea97ea7be8e8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=36.1 K 2024-11-13T03:52:23,484 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49] 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97cd4f810d647079d52ea97ea7be8e8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d2c08438d346f58661248849fa6793, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42317b1ab2fc4c90808c0eccf6d5b148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731469938193 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f5c786dcd143419f3328e6489377de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731469938193 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3b713c56724233b477ee32dddee539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469940619 2024-11-13T03:52:23,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9610d17cc80f41118cd7ca493f57ec49, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469940619 2024-11-13T03:52:23,492 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:23,493 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/55a6a511a7894dfa8e5b6620a34cec6c is 50, key is test_row_0/B:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:23,493 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:23,495 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111340625873c3284febb52d9deeaa4e6e96_42c4710928cbd1be72d51fa776aaa85f store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742244_1420 (size=12541) 2024-11-13T03:52:23,509 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111340625873c3284febb52d9deeaa4e6e96_42c4710928cbd1be72d51fa776aaa85f, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:23,510 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111340625873c3284febb52d9deeaa4e6e96_42c4710928cbd1be72d51fa776aaa85f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:23,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742245_1421 (size=4469) 2024-11-13T03:52:23,907 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/55a6a511a7894dfa8e5b6620a34cec6c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/55a6a511a7894dfa8e5b6620a34cec6c 2024-11-13T03:52:23,914 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/B of 42c4710928cbd1be72d51fa776aaa85f into 55a6a511a7894dfa8e5b6620a34cec6c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:23,914 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:23,914 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/B, priority=13, startTime=1731469943483; duration=0sec 2024-11-13T03:52:23,914 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:23,914 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:B 2024-11-13T03:52:23,914 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:23,915 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:23,915 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:23,915 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 42c4710928cbd1be72d51fa776aaa85f/C is initiating minor compaction (all files) 2024-11-13T03:52:23,916 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42c4710928cbd1be72d51fa776aaa85f/C in TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:23,916 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/7eb6576021e64aababa9f601b162f875 is 175, key is test_row_0/A:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:23,916 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/32f8ecb669b7448f842ba18f7b30f53f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp, totalSize=36.1 K 2024-11-13T03:52:23,916 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f8ecb669b7448f842ba18f7b30f53f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469935906 2024-11-13T03:52:23,917 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e959a0281684a6cb739b94c42917f92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731469938193 2024-11-13T03:52:23,917 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9341d8cb464a49bb0d8881fc24ce54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469940619 2024-11-13T03:52:23,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742246_1422 (size=31495) 2024-11-13T03:52:23,926 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42c4710928cbd1be72d51fa776aaa85f#C#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:23,926 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/d31ee5ef24ac46a4a0e2a2e6e7d100f4 is 50, key is test_row_0/C:col10/1731469940628/Put/seqid=0 2024-11-13T03:52:23,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742247_1423 (size=12541) 2024-11-13T03:52:24,331 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/7eb6576021e64aababa9f601b162f875 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/7eb6576021e64aababa9f601b162f875 2024-11-13T03:52:24,338 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/d31ee5ef24ac46a4a0e2a2e6e7d100f4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d31ee5ef24ac46a4a0e2a2e6e7d100f4 2024-11-13T03:52:24,339 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/A of 42c4710928cbd1be72d51fa776aaa85f into 7eb6576021e64aababa9f601b162f875(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:24,339 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:24,339 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/A, priority=13, startTime=1731469943482; duration=0sec 2024-11-13T03:52:24,339 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:24,340 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:A 2024-11-13T03:52:24,342 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42c4710928cbd1be72d51fa776aaa85f/C of 42c4710928cbd1be72d51fa776aaa85f into d31ee5ef24ac46a4a0e2a2e6e7d100f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:24,342 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42c4710928cbd1be72d51fa776aaa85f: 2024-11-13T03:52:24,342 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., storeName=42c4710928cbd1be72d51fa776aaa85f/C, priority=13, startTime=1731469943483; duration=0sec 2024-11-13T03:52:24,342 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:24,342 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42c4710928cbd1be72d51fa776aaa85f:C 2024-11-13T03:52:31,379 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-13T03:52:31,379 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:31,379 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,380 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:31,389 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-13T03:52:31,389 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:31,389 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,390 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1174 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3515 rows 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1164 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3485 rows 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1165 2024-11-13T03:52:31,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3495 rows 2024-11-13T03:52:31,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-13T03:52:31,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3499 rows 2024-11-13T03:52:31,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1166 2024-11-13T03:52:31,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3490 rows 2024-11-13T03:52:31,391 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:52:31,391 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:52:31,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:31,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:31,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:52:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:31,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469951397"}]},"ts":"1731469951397"} 2024-11-13T03:52:31,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:52:31,399 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:52:31,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:52:31,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, UNASSIGN}] 2024-11-13T03:52:31,402 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, UNASSIGN 2024-11-13T03:52:31,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:31,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, UNASSIGN because future has completed 2024-11-13T03:52:31,405 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:52:31,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:52:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:31,558 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:31,558 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:52:31,559 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 42c4710928cbd1be72d51fa776aaa85f, disabling compactions & flushes 2024-11-13T03:52:31,559 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:31,559 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:31,559 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. after waiting 0 ms 2024-11-13T03:52:31,559 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:31,559 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 42c4710928cbd1be72d51fa776aaa85f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=A 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=B 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42c4710928cbd1be72d51fa776aaa85f, store=C 2024-11-13T03:52:31,560 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:31,571 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a1b44d810094455882d75699aa843f26_42c4710928cbd1be72d51fa776aaa85f is 50, key is test_row_0/A:col10/1731469942598/Put/seqid=0 2024-11-13T03:52:31,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742248_1424 (size=9914) 2024-11-13T03:52:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:31,975 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:31,981 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a1b44d810094455882d75699aa843f26_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1b44d810094455882d75699aa843f26_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:31,983 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6e1ac64e189740909251e972a8f9482b, store: [table=TestAcidGuarantees family=A region=42c4710928cbd1be72d51fa776aaa85f] 2024-11-13T03:52:31,983 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6e1ac64e189740909251e972a8f9482b is 175, key is test_row_0/A:col10/1731469942598/Put/seqid=0 2024-11-13T03:52:31,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742249_1425 (size=22561) 2024-11-13T03:52:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:32,389 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6e1ac64e189740909251e972a8f9482b 2024-11-13T03:52:32,402 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/6b933e667ad5456587ce15ef9c267b1f is 50, key is test_row_0/B:col10/1731469942598/Put/seqid=0 2024-11-13T03:52:32,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742250_1426 (size=9857) 2024-11-13T03:52:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:32,808 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/6b933e667ad5456587ce15ef9c267b1f 2024-11-13T03:52:32,816 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/8844d6c945ee40b08ae39f146dc83bf4 is 50, key is test_row_0/C:col10/1731469942598/Put/seqid=0 2024-11-13T03:52:32,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742251_1427 (size=9857) 2024-11-13T03:52:33,222 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/8844d6c945ee40b08ae39f146dc83bf4 2024-11-13T03:52:33,233 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/A/6e1ac64e189740909251e972a8f9482b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6e1ac64e189740909251e972a8f9482b 2024-11-13T03:52:33,237 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6e1ac64e189740909251e972a8f9482b, entries=100, sequenceid=327, filesize=22.0 K 2024-11-13T03:52:33,238 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/B/6b933e667ad5456587ce15ef9c267b1f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/6b933e667ad5456587ce15ef9c267b1f 2024-11-13T03:52:33,242 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/6b933e667ad5456587ce15ef9c267b1f, entries=100, sequenceid=327, filesize=9.6 K 2024-11-13T03:52:33,243 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/.tmp/C/8844d6c945ee40b08ae39f146dc83bf4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/8844d6c945ee40b08ae39f146dc83bf4 2024-11-13T03:52:33,247 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/8844d6c945ee40b08ae39f146dc83bf4, entries=100, sequenceid=327, filesize=9.6 K 2024-11-13T03:52:33,248 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42c4710928cbd1be72d51fa776aaa85f in 1689ms, sequenceid=327, compaction requested=false 2024-11-13T03:52:33,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49] to archive 2024-11-13T03:52:33,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:33,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/806a8c8b948d4778b2fc52da1f1007f9 2024-11-13T03:52:33,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/ace587e38bae44119735b1f1974f6200 2024-11-13T03:52:33,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/417c8d2328bc402dbec388f505fa7bdc 2024-11-13T03:52:33,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/8df8fab096ca4bdea551b72af4561cff 2024-11-13T03:52:33,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/c7d2c08438d346f58661248849fa6793 2024-11-13T03:52:33,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/d7f5c786dcd143419f3328e6489377de 2024-11-13T03:52:33,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/9610d17cc80f41118cd7ca493f57ec49 2024-11-13T03:52:33,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/64b55e9a784c4693b1ff346ce57a4482, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/b97cd4f810d647079d52ea97ea7be8e8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539] to archive 2024-11-13T03:52:33,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:33,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/64b55e9a784c4693b1ff346ce57a4482 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/64b55e9a784c4693b1ff346ce57a4482 2024-11-13T03:52:33,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/33be778cd58c489fbb8dce62adc665ec 2024-11-13T03:52:33,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/f06f0fd60e6d41fcadd80b12a48209c7 2024-11-13T03:52:33,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/b97cd4f810d647079d52ea97ea7be8e8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/b97cd4f810d647079d52ea97ea7be8e8 2024-11-13T03:52:33,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/3b6429a1e81d4978956b14dd01ac2e8b 2024-11-13T03:52:33,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/42317b1ab2fc4c90808c0eccf6d5b148 2024-11-13T03:52:33,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/cd3b713c56724233b477ee32dddee539 2024-11-13T03:52:33,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/042ae51bd45c4a888904a21985a5c586, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/32f8ecb669b7448f842ba18f7b30f53f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54] to archive 2024-11-13T03:52:33,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:52:33,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/042ae51bd45c4a888904a21985a5c586 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/042ae51bd45c4a888904a21985a5c586 2024-11-13T03:52:33,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/dd1ffbaf6b4149c1ae0078de5064a925 2024-11-13T03:52:33,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/da59fbd76e7643a184006ebd5718812c 2024-11-13T03:52:33,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/32f8ecb669b7448f842ba18f7b30f53f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/32f8ecb669b7448f842ba18f7b30f53f 2024-11-13T03:52:33,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/9176feee7bdc47a38763adfaf9a525c6 2024-11-13T03:52:33,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/1e959a0281684a6cb739b94c42917f92 2024-11-13T03:52:33,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/7c9341d8cb464a49bb0d8881fc24ce54 2024-11-13T03:52:33,279 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits/330.seqid, newMaxSeqId=330, maxSeqId=4 2024-11-13T03:52:33,280 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. 2024-11-13T03:52:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 42c4710928cbd1be72d51fa776aaa85f: Waiting for close lock at 1731469951558Running coprocessor pre-close hooks at 1731469951558Disabling compacts and flushes for region at 1731469951558Disabling writes for close at 1731469951559 (+1 ms)Obtaining lock to block concurrent updates at 1731469951559Preparing flush snapshotting stores in 42c4710928cbd1be72d51fa776aaa85f at 1731469951559Finished memstore snapshotting TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731469951560 (+1 ms)Flushing stores of TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f. at 1731469951562 (+2 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/A: creating writer at 1731469951562Flushing mob file 42c4710928cbd1be72d51fa776aaa85f/A: appending metadata at 1731469951570 (+8 ms)Flushing mob file 42c4710928cbd1be72d51fa776aaa85f/A: closing flushed file at 1731469951570Flushing 42c4710928cbd1be72d51fa776aaa85f/A: appending metadata at 1731469951983 (+413 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/A: closing flushed file at 1731469951983Flushing 42c4710928cbd1be72d51fa776aaa85f/B: creating writer at 1731469952398 (+415 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/B: appending metadata at 1731469952401 (+3 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/B: closing flushed file at 1731469952401Flushing 42c4710928cbd1be72d51fa776aaa85f/C: creating writer at 1731469952813 (+412 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/C: appending metadata at 1731469952816 (+3 ms)Flushing 42c4710928cbd1be72d51fa776aaa85f/C: closing flushed file at 1731469952816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ab1894: reopening flushed file at 1731469953231 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c6f0e77: reopening flushed file at 1731469953237 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59f50d43: reopening flushed file at 1731469953242 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42c4710928cbd1be72d51fa776aaa85f in 1689ms, sequenceid=327, compaction requested=false at 1731469953248 (+6 ms)Writing region close event to WAL at 1731469953276 (+28 ms)Running coprocessor post-close hooks at 1731469953280 (+4 ms)Closed at 1731469953280 2024-11-13T03:52:33,282 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,282 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=42c4710928cbd1be72d51fa776aaa85f, regionState=CLOSED 2024-11-13T03:52:33,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:52:33,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T03:52:33,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 42c4710928cbd1be72d51fa776aaa85f, server=ac1a51731247,42123,1731469817167 in 1.8790 sec 2024-11-13T03:52:33,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-13T03:52:33,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42c4710928cbd1be72d51fa776aaa85f, UNASSIGN in 1.8850 sec 2024-11-13T03:52:33,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T03:52:33,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-13T03:52:33,290 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469953290"}]},"ts":"1731469953290"} 2024-11-13T03:52:33,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:52:33,291 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:52:33,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8980 sec 2024-11-13T03:52:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T03:52:33,548 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:33,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:52:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,554 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T03:52:33,556 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,559 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,562 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits] 2024-11-13T03:52:33,566 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6e1ac64e189740909251e972a8f9482b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/6e1ac64e189740909251e972a8f9482b 2024-11-13T03:52:33,567 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/7eb6576021e64aababa9f601b162f875 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/A/7eb6576021e64aababa9f601b162f875 2024-11-13T03:52:33,571 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/55a6a511a7894dfa8e5b6620a34cec6c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/55a6a511a7894dfa8e5b6620a34cec6c 2024-11-13T03:52:33,573 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/6b933e667ad5456587ce15ef9c267b1f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/B/6b933e667ad5456587ce15ef9c267b1f 2024-11-13T03:52:33,576 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/8844d6c945ee40b08ae39f146dc83bf4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/8844d6c945ee40b08ae39f146dc83bf4 2024-11-13T03:52:33,577 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d31ee5ef24ac46a4a0e2a2e6e7d100f4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/C/d31ee5ef24ac46a4a0e2a2e6e7d100f4 2024-11-13T03:52:33,579 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits/330.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f/recovered.edits/330.seqid 2024-11-13T03:52:33,580 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,580 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:52:33,580 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:52:33,581 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T03:52:33,584 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130788c909ecf949bd83a91b7c3976b1d1_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130788c909ecf949bd83a91b7c3976b1d1_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,585 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130fcaf0409f5b44d0a2dd390bd4878e35_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130fcaf0409f5b44d0a2dd390bd4878e35_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,586 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131457a20041ae4b66805c49b1fab6aa6c_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131457a20041ae4b66805c49b1fab6aa6c_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,587 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113250ef05a4c794fa0b6fc21794e70d2cd_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113250ef05a4c794fa0b6fc21794e70d2cd_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,587 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133539737815874fb18e85c5302a098487_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133539737815874fb18e85c5302a098487_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,588 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335641fc2cf444220b2286023646de17d_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335641fc2cf444220b2286023646de17d_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,589 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343820fc8ada9406e927a501474e4db2c_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343820fc8ada9406e927a501474e4db2c_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,590 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b35b041e37148fa90fa362d6914019e_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b35b041e37148fa90fa362d6914019e_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,590 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113701294265fa944a89ba4fa4dd8e52240_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113701294265fa944a89ba4fa4dd8e52240_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,591 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113802c86a5fd0047049d5f699fb770a05d_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113802c86a5fd0047049d5f699fb770a05d_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,592 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113938c05c2731e411a990557ef93c6f974_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113938c05c2731e411a990557ef93c6f974_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,592 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111396c5ae0467aa4b779e337b4c6681ed07_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111396c5ae0467aa4b779e337b4c6681ed07_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,593 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1b44d810094455882d75699aa843f26_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1b44d810094455882d75699aa843f26_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,594 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b164342822774c3f997be71d01e69f6b_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b164342822774c3f997be71d01e69f6b_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,595 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c476569d90414a79a52bcde30a6d9e26_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c476569d90414a79a52bcde30a6d9e26_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,595 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de89035291874995b417f2c159cf7f75_42c4710928cbd1be72d51fa776aaa85f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de89035291874995b417f2c159cf7f75_42c4710928cbd1be72d51fa776aaa85f 2024-11-13T03:52:33,596 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:52:33,597 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,599 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:52:33,601 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:52:33,602 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,602 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:52:33,602 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731469953602"}]},"ts":"9223372036854775807"} 2024-11-13T03:52:33,604 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:52:33,604 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 42c4710928cbd1be72d51fa776aaa85f, NAME => 'TestAcidGuarantees,,1731469917516.42c4710928cbd1be72d51fa776aaa85f.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:52:33,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:52:33,604 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731469953604"}]},"ts":"9223372036854775807"} 2024-11-13T03:52:33,606 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:52:33,606 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-13T03:52:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T03:52:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:52:33,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:33,676 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 249), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 260) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8145 (was 8178) 2024-11-13T03:52:33,684 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=8145 2024-11-13T03:52:33,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:52:33,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:52:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:33,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:52:33,687 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:33,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-13T03:52:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:33,688 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:52:33,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742252_1428 (size=1061) 2024-11-13T03:52:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:34,098 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:52:34,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742253_1429 (size=53) 2024-11-13T03:52:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:34,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:52:34,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 489c6f15a08a8180330774d57ac110a8, disabling compactions & flushes 2024-11-13T03:52:34,511 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. after waiting 0 ms 2024-11-13T03:52:34,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,512 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 489c6f15a08a8180330774d57ac110a8: Waiting for close lock at 1731469954511Disabling compacts and flushes for region at 1731469954511Disabling writes for close at 1731469954512 (+1 ms)Writing region close event to WAL at 1731469954512Closed at 1731469954512 2024-11-13T03:52:34,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:52:34,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469954515"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469954515"}]},"ts":"1731469954515"} 2024-11-13T03:52:34,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:52:34,518 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:52:34,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469954518"}]},"ts":"1731469954518"} 2024-11-13T03:52:34,520 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:52:34,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, ASSIGN}] 2024-11-13T03:52:34,522 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, ASSIGN 2024-11-13T03:52:34,524 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:52:34,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=489c6f15a08a8180330774d57ac110a8, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:34,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, ASSIGN because future has completed 2024-11-13T03:52:34,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:52:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:34,840 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,841 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:52:34,842 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,843 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:52:34,843 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,843 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,845 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,847 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:34,847 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489c6f15a08a8180330774d57ac110a8 columnFamilyName A 2024-11-13T03:52:34,847 DEBUG [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:34,848 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(327): Store=489c6f15a08a8180330774d57ac110a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:34,848 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,850 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:34,850 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489c6f15a08a8180330774d57ac110a8 columnFamilyName B 2024-11-13T03:52:34,850 DEBUG [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:34,851 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(327): Store=489c6f15a08a8180330774d57ac110a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:34,851 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,852 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:52:34,852 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489c6f15a08a8180330774d57ac110a8 columnFamilyName C 2024-11-13T03:52:34,852 DEBUG [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:52:34,853 INFO [StoreOpener-489c6f15a08a8180330774d57ac110a8-1 {}] regionserver.HStore(327): Store=489c6f15a08a8180330774d57ac110a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:52:34,853 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,853 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,853 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,854 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,855 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,855 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,855 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:52:34,857 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,858 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:52:34,859 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 489c6f15a08a8180330774d57ac110a8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62149311, jitterRate=-0.07390309870243073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:52:34,859 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:34,860 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 489c6f15a08a8180330774d57ac110a8: Running coprocessor pre-open hook at 1731469954843Writing region info on filesystem at 1731469954843Initializing all the Stores at 1731469954845 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469954845Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469954845Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469954845Cleaning up temporary data from old regions at 1731469954855 (+10 ms)Running coprocessor post-open hooks at 1731469954859 (+4 ms)Region opened successfully at 1731469954860 (+1 ms) 2024-11-13T03:52:34,860 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., pid=120, masterSystemTime=1731469954833 2024-11-13T03:52:34,862 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,862 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:34,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=489c6f15a08a8180330774d57ac110a8, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:34,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:52:34,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T03:52:34,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 in 186 msec 2024-11-13T03:52:34,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-13T03:52:34,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, ASSIGN in 346 msec 2024-11-13T03:52:34,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:52:34,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469954869"}]},"ts":"1731469954869"} 2024-11-13T03:52:34,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:52:34,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:52:34,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-13T03:52:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T03:52:35,838 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:35,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,844 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a71b81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3dacca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,849 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,852 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:52:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:52:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@532a0940 2024-11-13T03:52:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:52:35,854 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:52:35,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,857 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cf3a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42952b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,861 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e4b94a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,863 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6f3740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,865 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8baf1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,867 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb0bca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375008a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,869 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c955a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c24646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30beebba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a6cd01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130f8b34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,876 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270bb0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,877 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb43e52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6846fe5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,882 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42577cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a0577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:52:35,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:52:35,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:52:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:52:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:52:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540285e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:52:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:52:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:35,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:52:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da0a0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:52:35,887 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,887 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,887 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,888 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:35,888 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,888 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,889 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T03:52:35,889 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,890 DEBUG [Thread-1944 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,890 DEBUG [Thread-1944 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:35,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,893 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,894 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:52:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:35,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:52:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:52:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:52:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2] 2024-11-13T03:52:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:35,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:35,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/0a94486eb5da4d7ab6e4ddc22e15f9bc is 50, key is test_row_0/A:col10/1731469955894/Put/seqid=0 2024-11-13T03:52:35,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742254_1430 (size=12001) 2024-11-13T03:52:35,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469955913, max timestamp=1731469955936Num uniques 859; 2024-11-13T03:52:35,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:35,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469955913, max timestamp=1731469955936Num uniques 859; 2024-11-13T03:52:35,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:35,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469955913, max timestamp=1731469955936Num uniques 859; 2024-11-13T03:52:35,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:35,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/0a94486eb5da4d7ab6e4ddc22e15f9bc 2024-11-13T03:52:35,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7d048dff69c742459a67801d651e62e3 is 50, key is test_row_0/B:col10/1731469955894/Put/seqid=0 2024-11-13T03:52:35,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742255_1431 (size=12001) 2024-11-13T03:52:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470015970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470015971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470015973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470015977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470015982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:36,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470016087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470016087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470016087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470016087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470016097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470016316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7d048dff69c742459a67801d651e62e3 2024-11-13T03:52:36,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/a64fbf098ed243a39e9559d887f27130 is 50, key is test_row_0/C:col10/1731469955894/Put/seqid=0 2024-11-13T03:52:36,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742256_1432 (size=12001) 2024-11-13T03:52:36,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470016617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470016627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470016627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470016627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470016636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:36,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/a64fbf098ed243a39e9559d887f27130 2024-11-13T03:52:36,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/0a94486eb5da4d7ab6e4ddc22e15f9bc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc 2024-11-13T03:52:36,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T03:52:36,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7d048dff69c742459a67801d651e62e3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3 2024-11-13T03:52:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:36,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T03:52:36,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/a64fbf098ed243a39e9559d887f27130 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130 2024-11-13T03:52:36,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T03:52:36,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 489c6f15a08a8180330774d57ac110a8 in 922ms, sequenceid=11, compaction requested=false 2024-11-13T03:52:36,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:36,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T03:52:36,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:36,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:36,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/34b36471750e4ffaa35ff755ae1b0c42 is 50, key is test_row_0/A:col10/1731469955938/Put/seqid=0 2024-11-13T03:52:36,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742257_1433 (size=12001) 2024-11-13T03:52:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470017144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470017145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470017146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470017147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470017147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470017256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470017257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470017267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470017267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/34b36471750e4ffaa35ff755ae1b0c42 2024-11-13T03:52:37,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/58fcac5e6253482b8e3945da996404c2 is 50, key is test_row_0/B:col10/1731469955938/Put/seqid=0 2024-11-13T03:52:37,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742258_1434 (size=12001) 2024-11-13T03:52:37,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470017467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470017467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470017477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470017477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/58fcac5e6253482b8e3945da996404c2 2024-11-13T03:52:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470017776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470017776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/e6f6e4caee7b40d7abd47bc8ad996b17 is 50, key is test_row_0/C:col10/1731469955938/Put/seqid=0 2024-11-13T03:52:37,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742259_1435 (size=12001) 2024-11-13T03:52:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470017797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470017797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:38,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470018167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,176 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/e6f6e4caee7b40d7abd47bc8ad996b17 2024-11-13T03:52:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/34b36471750e4ffaa35ff755ae1b0c42 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42 2024-11-13T03:52:38,200 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T03:52:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/58fcac5e6253482b8e3945da996404c2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2 2024-11-13T03:52:38,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T03:52:38,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/e6f6e4caee7b40d7abd47bc8ad996b17 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17 2024-11-13T03:52:38,206 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T03:52:38,207 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 489c6f15a08a8180330774d57ac110a8 in 1243ms, sequenceid=46, compaction requested=false 2024-11-13T03:52:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T03:52:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T03:52:38,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T03:52:38,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-13T03:52:38,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3220 sec 2024-11-13T03:52:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:38,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:52:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:38,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:38,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:38,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:38,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/6be6b174a03a48d69d1db11d0a7286c6 is 50, key is test_row_0/A:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:38,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742260_1436 (size=14341) 2024-11-13T03:52:38,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/6be6b174a03a48d69d1db11d0a7286c6 2024-11-13T03:52:38,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/daee1df01a74422884230cca03ee286b is 50, key is test_row_0/B:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:38,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742261_1437 (size=12001) 2024-11-13T03:52:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470018348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469958341, max timestamp=1731469958356Num uniques 859; 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469958341, max timestamp=1731469958356Num uniques 859; 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469958341, max timestamp=1731469958356Num uniques 859; 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:38,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470018397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470018400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470018407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470018467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470018516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470018516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470018516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470018677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/daee1df01a74422884230cca03ee286b 2024-11-13T03:52:38,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c8f722f99d014facb1c5afb4cba7538e is 50, key is test_row_0/C:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742262_1438 (size=12001) 2024-11-13T03:52:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470018736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470018736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470018737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:38,806 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:52:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470018997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470019057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470019057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470019057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c8f722f99d014facb1c5afb4cba7538e 2024-11-13T03:52:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/6be6b174a03a48d69d1db11d0a7286c6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6 2024-11-13T03:52:39,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6, entries=200, sequenceid=57, filesize=14.0 K 2024-11-13T03:52:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/daee1df01a74422884230cca03ee286b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b 2024-11-13T03:52:39,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-13T03:52:39,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c8f722f99d014facb1c5afb4cba7538e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e 2024-11-13T03:52:39,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e, entries=150, sequenceid=57, filesize=11.7 K 2024-11-13T03:52:39,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 489c6f15a08a8180330774d57ac110a8 in 855ms, sequenceid=57, compaction requested=true 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:39,156 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:39,156 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:39,157 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:39,157 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:39,157 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:39,157 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:39,157 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:39,157 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=37.4 K 2024-11-13T03:52:39,158 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:39,158 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=35.2 K 2024-11-13T03:52:39,158 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d048dff69c742459a67801d651e62e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469955894 2024-11-13T03:52:39,158 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a94486eb5da4d7ab6e4ddc22e15f9bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469955894 2024-11-13T03:52:39,158 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 58fcac5e6253482b8e3945da996404c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731469955938 2024-11-13T03:52:39,158 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b36471750e4ffaa35ff755ae1b0c42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731469955938 2024-11-13T03:52:39,159 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting daee1df01a74422884230cca03ee286b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:39,159 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be6b174a03a48d69d1db11d0a7286c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:39,164 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:39,164 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/67cbddbc7caf4721a05de4bd97fc24a8 is 50, key is test_row_0/B:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:39,165 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:39,165 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/c650cf1fba444b9fb2cd6f391e937a65 is 50, key is test_row_0/A:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742263_1439 (size=12104) 2024-11-13T03:52:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742264_1440 (size=12104) 2024-11-13T03:52:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:39,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/85e94f420c3c4fecab40fe558326ea6f is 50, key is test_row_0/A:col10/1731469958396/Put/seqid=0 2024-11-13T03:52:39,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742265_1441 (size=14341) 2024-11-13T03:52:39,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/85e94f420c3c4fecab40fe558326ea6f 2024-11-13T03:52:39,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/162307b5bcf745c79051d4661c2dd959 is 50, key is test_row_0/B:col10/1731469958396/Put/seqid=0 2024-11-13T03:52:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742266_1442 (size=12001) 2024-11-13T03:52:39,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470019561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,578 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/67cbddbc7caf4721a05de4bd97fc24a8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/67cbddbc7caf4721a05de4bd97fc24a8 2024-11-13T03:52:39,582 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into 67cbddbc7caf4721a05de4bd97fc24a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:39,582 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:39,582 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=13, startTime=1731469959156; duration=0sec 2024-11-13T03:52:39,582 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:39,582 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:39,582 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:39,583 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:39,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:39,584 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:39,584 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=35.2 K 2024-11-13T03:52:39,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a64fbf098ed243a39e9559d887f27130, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731469955894 2024-11-13T03:52:39,584 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f6e4caee7b40d7abd47bc8ad996b17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731469955938 2024-11-13T03:52:39,585 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/c650cf1fba444b9fb2cd6f391e937a65 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c650cf1fba444b9fb2cd6f391e937a65 2024-11-13T03:52:39,585 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f722f99d014facb1c5afb4cba7538e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:39,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470019577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470019577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470019577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,588 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into c650cf1fba444b9fb2cd6f391e937a65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:39,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:39,588 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=13, startTime=1731469959156; duration=0sec 2024-11-13T03:52:39,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:39,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:39,593 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:39,593 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/ffa16d594606457db3e48aa2ce4446d9 is 50, key is test_row_0/C:col10/1731469957139/Put/seqid=0 2024-11-13T03:52:39,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742267_1443 (size=12104) 2024-11-13T03:52:39,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470019676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470019887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:39,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/162307b5bcf745c79051d4661c2dd959 2024-11-13T03:52:39,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d8ec1e1c47ee451c91277017d866298b is 50, key is test_row_0/C:col10/1731469958396/Put/seqid=0 2024-11-13T03:52:39,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742268_1444 (size=12001) 2024-11-13T03:52:40,003 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/ffa16d594606457db3e48aa2ce4446d9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/ffa16d594606457db3e48aa2ce4446d9 2024-11-13T03:52:40,007 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into ffa16d594606457db3e48aa2ce4446d9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:40,007 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:40,007 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=13, startTime=1731469959156; duration=0sec 2024-11-13T03:52:40,007 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:40,007 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T03:52:40,067 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:40,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T03:52:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:40,069 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:40,069 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:40,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470020197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,202 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470020207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T03:52:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:40,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:40,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d8ec1e1c47ee451c91277017d866298b 2024-11-13T03:52:40,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/85e94f420c3c4fecab40fe558326ea6f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f 2024-11-13T03:52:40,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f, entries=200, sequenceid=93, filesize=14.0 K 2024-11-13T03:52:40,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/162307b5bcf745c79051d4661c2dd959 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959 2024-11-13T03:52:40,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T03:52:40,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d8ec1e1c47ee451c91277017d866298b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b 2024-11-13T03:52:40,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T03:52:40,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 489c6f15a08a8180330774d57ac110a8 in 840ms, sequenceid=93, compaction requested=false 2024-11-13T03:52:40,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:40,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:40,374 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:40,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/f3190f6e8af740b4bf26281966b76bae is 50, key is test_row_0/A:col10/1731469959560/Put/seqid=0 2024-11-13T03:52:40,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742269_1445 (size=12001) 2024-11-13T03:52:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:40,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469960609, max timestamp=1731469960642Num uniques 859; 2024-11-13T03:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469960609, max timestamp=1731469960642Num uniques 859; 2024-11-13T03:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469960609, max timestamp=1731469960642Num uniques 859; 2024-11-13T03:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470020686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470020690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470020691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470020726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,784 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/f3190f6e8af740b4bf26281966b76bae 2024-11-13T03:52:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/e47887b8e5b34d90845431751103815a is 50, key is test_row_0/B:col10/1731469959560/Put/seqid=0 2024-11-13T03:52:40,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742270_1446 (size=12001) 2024-11-13T03:52:40,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470020797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470020807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470020807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470021007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470021027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470021027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,192 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/e47887b8e5b34d90845431751103815a 2024-11-13T03:52:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/5bbf6e6dc310403d87a1645861dc7128 is 50, key is test_row_0/C:col10/1731469959560/Put/seqid=0 2024-11-13T03:52:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742271_1447 (size=12001) 2024-11-13T03:52:41,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470021327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470021337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470021337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/5bbf6e6dc310403d87a1645861dc7128 2024-11-13T03:52:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/f3190f6e8af740b4bf26281966b76bae as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae 2024-11-13T03:52:41,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae, entries=150, sequenceid=106, filesize=11.7 K 2024-11-13T03:52:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/e47887b8e5b34d90845431751103815a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a 2024-11-13T03:52:41,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a, entries=150, sequenceid=106, filesize=11.7 K 2024-11-13T03:52:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/5bbf6e6dc310403d87a1645861dc7128 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128 2024-11-13T03:52:41,648 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128, entries=150, sequenceid=106, filesize=11.7 K 2024-11-13T03:52:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 489c6f15a08a8180330774d57ac110a8 in 1275ms, sequenceid=106, compaction requested=true 2024-11-13T03:52:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T03:52:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T03:52:41,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T03:52:41,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-11-13T03:52:41,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5850 sec 2024-11-13T03:52:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:41,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:41,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:41,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/08fa86101090467299b76038e2d3e2e4 is 50, key is test_row_0/A:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:41,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742272_1448 (size=14541) 2024-11-13T03:52:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470021798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470021837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470021857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470021857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470021917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470022126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/08fa86101090467299b76038e2d3e2e4 2024-11-13T03:52:42,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/22240effb367405485565b707509353d is 50, key is test_row_0/B:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742273_1449 (size=12151) 2024-11-13T03:52:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T03:52:42,217 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:42,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T03:52:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:42,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:42,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:42,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:42,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470022437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:42,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/22240effb367405485565b707509353d 2024-11-13T03:52:42,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/47e4ebc4c00a474b82f07164818aa21a is 50, key is test_row_0/C:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:42,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742274_1450 (size=12151) 2024-11-13T03:52:42,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470022857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,867 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470022866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,876 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470022877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,884 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:42,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/47e4ebc4c00a474b82f07164818aa21a 2024-11-13T03:52:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:42,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/08fa86101090467299b76038e2d3e2e4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4 2024-11-13T03:52:42,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4, entries=200, sequenceid=143, filesize=14.2 K 2024-11-13T03:52:42,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/22240effb367405485565b707509353d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d 2024-11-13T03:52:42,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T03:52:42,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/47e4ebc4c00a474b82f07164818aa21a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a 2024-11-13T03:52:42,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T03:52:42,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 489c6f15a08a8180330774d57ac110a8 in 1242ms, sequenceid=143, compaction requested=true 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:42,996 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:42,996 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:42,997 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,997 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:42,997 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c650cf1fba444b9fb2cd6f391e937a65, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=51.7 K 2024-11-13T03:52:42,997 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/67cbddbc7caf4721a05de4bd97fc24a8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=47.1 K 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c650cf1fba444b9fb2cd6f391e937a65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 67cbddbc7caf4721a05de4bd97fc24a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e94f420c3c4fecab40fe558326ea6f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731469958396 2024-11-13T03:52:42,997 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 162307b5bcf745c79051d4661c2dd959, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731469958396 2024-11-13T03:52:42,998 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3190f6e8af740b4bf26281966b76bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731469959534 2024-11-13T03:52:42,998 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e47887b8e5b34d90845431751103815a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731469959534 2024-11-13T03:52:42,998 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08fa86101090467299b76038e2d3e2e4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:42,998 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 22240effb367405485565b707509353d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:43,005 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:43,005 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/c498d613b4264ffd9b84151d0116d0cf is 50, key is test_row_0/A:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:43,007 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:43,008 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/a28a497c0c084c09911c641fac727ea7 is 50, key is test_row_0/B:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:43,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742275_1451 (size=12391) 2024-11-13T03:52:43,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742276_1452 (size=12391) 2024-11-13T03:52:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T03:52:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:43,134 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T03:52:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:43,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/11a37af29ecb407fad7932153941f00b is 50, key is test_row_0/A:col10/1731469961797/Put/seqid=0 2024-11-13T03:52:43,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742277_1453 (size=12151) 2024-11-13T03:52:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:43,420 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/c498d613b4264ffd9b84151d0116d0cf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c498d613b4264ffd9b84151d0116d0cf 2024-11-13T03:52:43,421 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/a28a497c0c084c09911c641fac727ea7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a28a497c0c084c09911c641fac727ea7 2024-11-13T03:52:43,423 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into c498d613b4264ffd9b84151d0116d0cf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:43,423 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:43,423 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=12, startTime=1731469962996; duration=0sec 2024-11-13T03:52:43,423 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:43,423 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:43,423 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:43,424 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:43,424 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into a28a497c0c084c09911c641fac727ea7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:43,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:43,424 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:43,424 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=12, startTime=1731469962996; duration=0sec 2024-11-13T03:52:43,424 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:43,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:43,424 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:43,425 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/ffa16d594606457db3e48aa2ce4446d9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=47.1 K 2024-11-13T03:52:43,425 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa16d594606457db3e48aa2ce4446d9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731469957139 2024-11-13T03:52:43,425 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8ec1e1c47ee451c91277017d866298b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731469958396 2024-11-13T03:52:43,425 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbf6e6dc310403d87a1645861dc7128, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731469959534 2024-11-13T03:52:43,426 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e4ebc4c00a474b82f07164818aa21a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:43,436 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#384 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:43,436 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/dfa794c1877f4f60b757e5b6006c6bfd is 50, key is test_row_0/C:col10/1731469960684/Put/seqid=0 2024-11-13T03:52:43,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742278_1454 (size=12391) 2024-11-13T03:52:43,543 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/11a37af29ecb407fad7932153941f00b 2024-11-13T03:52:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/318b83616b2c415ebbd77f0f1dbf2c84 is 50, key is test_row_0/B:col10/1731469961797/Put/seqid=0 2024-11-13T03:52:43,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742279_1455 (size=12151) 2024-11-13T03:52:43,852 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/dfa794c1877f4f60b757e5b6006c6bfd as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/dfa794c1877f4f60b757e5b6006c6bfd 2024-11-13T03:52:43,856 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into dfa794c1877f4f60b757e5b6006c6bfd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:43,856 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:43,856 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=12, startTime=1731469962996; duration=0sec 2024-11-13T03:52:43,856 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:43,856 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:43,964 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/318b83616b2c415ebbd77f0f1dbf2c84 2024-11-13T03:52:43,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/27fb3286a0684721b6e594228732f097 is 50, key is test_row_0/C:col10/1731469961797/Put/seqid=0 2024-11-13T03:52:43,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742280_1456 (size=12151) 2024-11-13T03:52:44,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469964040, max timestamp=1731469964059Num uniques 859; 2024-11-13T03:52:44,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:44,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469964040, max timestamp=1731469964059Num uniques 859; 2024-11-13T03:52:44,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:44,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469964040, max timestamp=1731469964059Num uniques 859; 2024-11-13T03:52:44,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:44,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731469964128, max timestamp=1731469964157Num uniques 1018; 2024-11-13T03:52:44,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731469964128, max timestamp=1731469964157Num uniques 1018; 2024-11-13T03:52:44,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731469964128, max timestamp=1731469964157Num uniques 1018; 2024-11-13T03:52:44,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T03:52:44,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T03:52:44,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T03:52:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470024254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731469964222, max timestamp=1731469964263Num uniques 1027; 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731469964222, max timestamp=1731469964263Num uniques 1027; 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731469964222, max timestamp=1731469964263Num uniques 1027; 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T03:52:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T03:52:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:44,376 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/27fb3286a0684721b6e594228732f097 2024-11-13T03:52:44,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731469964332, max timestamp=1731469964375Num uniques 1036; 2024-11-13T03:52:44,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731469964332, max timestamp=1731469964375Num uniques 1036; 2024-11-13T03:52:44,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T03:52:44,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T03:52:44,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731469964332, max timestamp=1731469964375Num uniques 1036; 2024-11-13T03:52:44,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T03:52:44,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/11a37af29ecb407fad7932153941f00b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b 2024-11-13T03:52:44,383 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T03:52:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/318b83616b2c415ebbd77f0f1dbf2c84 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84 2024-11-13T03:52:44,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T03:52:44,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/27fb3286a0684721b6e594228732f097 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097 2024-11-13T03:52:44,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T03:52:44,392 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=46.96 KB/48090 for 489c6f15a08a8180330774d57ac110a8 in 1258ms, sequenceid=152, compaction requested=false 2024-11-13T03:52:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T03:52:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T03:52:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:44,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=53.67 KB heapSize=138.70 KB 2024-11-13T03:52:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:44,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:44,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:44,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/595e435c25ea4aeeac6def397ef3aeb8 is 50, key is test_row_0/A:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:44,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T03:52:44,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-13T03:52:44,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1810 sec 2024-11-13T03:52:44,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742281_1457 (size=16931) 2024-11-13T03:52:44,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469964435, max timestamp=1731469964467Num uniques 859; 2024-11-13T03:52:44,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:44,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469964435, max timestamp=1731469964467Num uniques 859; 2024-11-13T03:52:44,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:44,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469964435, max timestamp=1731469964467Num uniques 859; 2024-11-13T03:52:44,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470024542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470024543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470024657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470024657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/595e435c25ea4aeeac6def397ef3aeb8 2024-11-13T03:52:44,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/708231201b1e49e1bc4620056094ad5c is 50, key is test_row_0/B:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:44,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742282_1458 (size=12151) 2024-11-13T03:52:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470024877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470024877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470024877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470024887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,898 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470024897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:44,906 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470025197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470025197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/708231201b1e49e1bc4620056094ad5c 2024-11-13T03:52:45,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c925d1ad72f440d388f278178d6f4fc0 is 50, key is test_row_0/C:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:45,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742283_1459 (size=12151) 2024-11-13T03:52:45,562 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:52:45,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c925d1ad72f440d388f278178d6f4fc0 2024-11-13T03:52:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/595e435c25ea4aeeac6def397ef3aeb8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8 2024-11-13T03:52:45,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8, entries=250, sequenceid=230, filesize=16.5 K 2024-11-13T03:52:45,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/708231201b1e49e1bc4620056094ad5c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c 2024-11-13T03:52:45,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T03:52:45,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/c925d1ad72f440d388f278178d6f4fc0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0 2024-11-13T03:52:45,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T03:52:45,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~138.66 KB/141984, currentSize=134.18 KB/137400 for 489c6f15a08a8180330774d57ac110a8 in 1248ms, sequenceid=230, compaction requested=true 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:45,642 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:45,642 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:45,643 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:45,643 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:45,643 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:45,644 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a28a497c0c084c09911c641fac727ea7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=35.8 K 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a28a497c0c084c09911c641fac727ea7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:45,644 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:45,644 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c498d613b4264ffd9b84151d0116d0cf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=40.5 K 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 318b83616b2c415ebbd77f0f1dbf2c84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731469961765 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c498d613b4264ffd9b84151d0116d0cf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 708231201b1e49e1bc4620056094ad5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964375 2024-11-13T03:52:45,644 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a37af29ecb407fad7932153941f00b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731469961765 2024-11-13T03:52:45,645 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595e435c25ea4aeeac6def397ef3aeb8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964362 2024-11-13T03:52:45,650 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:45,650 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/be7fdd9c30904dbf8340763c6fcfce89 is 50, key is test_row_0/B:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:45,653 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:45,653 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/21be84d39536437b9107ceef38756239 is 50, key is test_row_0/A:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:45,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742284_1460 (size=12493) 2024-11-13T03:52:45,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742285_1461 (size=12493) 2024-11-13T03:52:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:45,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:45,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/b0b1f8dc01c94d62b0ee99539b59440d is 50, key is test_row_0/A:col10/1731469964541/Put/seqid=0 2024-11-13T03:52:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742286_1462 (size=14741) 2024-11-13T03:52:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470025737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470025750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470025856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470025856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:46,068 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/be7fdd9c30904dbf8340763c6fcfce89 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/be7fdd9c30904dbf8340763c6fcfce89 2024-11-13T03:52:46,070 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/21be84d39536437b9107ceef38756239 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/21be84d39536437b9107ceef38756239 2024-11-13T03:52:46,071 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into be7fdd9c30904dbf8340763c6fcfce89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:46,071 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:46,071 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=13, startTime=1731469965642; duration=0sec 2024-11-13T03:52:46,072 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:46,072 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:46,072 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:46,072 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:46,072 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:46,072 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:46,072 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/dfa794c1877f4f60b757e5b6006c6bfd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=35.8 K 2024-11-13T03:52:46,073 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa794c1877f4f60b757e5b6006c6bfd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731469960673 2024-11-13T03:52:46,073 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fb3286a0684721b6e594228732f097, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731469961765 2024-11-13T03:52:46,073 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c925d1ad72f440d388f278178d6f4fc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964375 2024-11-13T03:52:46,074 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into 21be84d39536437b9107ceef38756239(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:46,074 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:46,074 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=13, startTime=1731469965642; duration=0sec 2024-11-13T03:52:46,074 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:46,074 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:46,078 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:46,079 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/1861c4068c1543c79b643eb4a4cac53d is 50, key is test_row_0/C:col10/1731469964388/Put/seqid=0 2024-11-13T03:52:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470026067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470026067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:46,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742287_1463 (size=12493) 2024-11-13T03:52:46,086 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/1861c4068c1543c79b643eb4a4cac53d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/1861c4068c1543c79b643eb4a4cac53d 2024-11-13T03:52:46,090 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into 1861c4068c1543c79b643eb4a4cac53d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:46,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:46,090 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=13, startTime=1731469965642; duration=0sec 2024-11-13T03:52:46,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:46,090 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:46,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/b0b1f8dc01c94d62b0ee99539b59440d 2024-11-13T03:52:46,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/0a3f8440ac7e45c88c74516086b3e6a6 is 50, key is test_row_0/B:col10/1731469964541/Put/seqid=0 2024-11-13T03:52:46,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742288_1464 (size=12301) 2024-11-13T03:52:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T03:52:46,387 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:46,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T03:52:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:46,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:46,389 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:46,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470026386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470026386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:46,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/0a3f8440ac7e45c88c74516086b3e6a6 2024-11-13T03:52:46,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T03:52:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:46,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:46,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/3633cf6820b247cb898d06a5a84cbde8 is 50, key is test_row_0/C:col10/1731469964541/Put/seqid=0 2024-11-13T03:52:46,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742289_1465 (size=12301) 2024-11-13T03:52:46,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/3633cf6820b247cb898d06a5a84cbde8 2024-11-13T03:52:46,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/b0b1f8dc01c94d62b0ee99539b59440d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d 2024-11-13T03:52:46,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d, entries=200, sequenceid=267, filesize=14.4 K 2024-11-13T03:52:46,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/0a3f8440ac7e45c88c74516086b3e6a6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6 2024-11-13T03:52:46,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T03:52:46,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/3633cf6820b247cb898d06a5a84cbde8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8 2024-11-13T03:52:46,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T03:52:46,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 489c6f15a08a8180330774d57ac110a8 in 846ms, sequenceid=267, compaction requested=false 2024-11-13T03:52:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:46,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:46,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/e28663bb8b3947d1ae85405b2c2ef941 is 50, key is test_row_0/A:col10/1731469965748/Put/seqid=0 2024-11-13T03:52:46,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742290_1466 (size=12301) 2024-11-13T03:52:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:46,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:46,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469966935, max timestamp=1731469966964Num uniques 859; 2024-11-13T03:52:46,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:46,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469966935, max timestamp=1731469966964Num uniques 859; 2024-11-13T03:52:46,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:46,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469966935, max timestamp=1731469966964Num uniques 859; 2024-11-13T03:52:46,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:52:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470027030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470027036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/e28663bb8b3947d1ae85405b2c2ef941 2024-11-13T03:52:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/9ae6f7eb33144cd98151b69566da7165 is 50, key is test_row_0/B:col10/1731469965748/Put/seqid=0 2024-11-13T03:52:47,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742291_1467 (size=12301) 2024-11-13T03:52:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470027147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470027147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470027357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470027357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,509 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/9ae6f7eb33144cd98151b69566da7165 2024-11-13T03:52:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/29b8d6d144494e27a863957bb0dd1d00 is 50, key is test_row_0/C:col10/1731469965748/Put/seqid=0 2024-11-13T03:52:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742292_1468 (size=12301) 2024-11-13T03:52:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470027677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470027677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:47,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/29b8d6d144494e27a863957bb0dd1d00 2024-11-13T03:52:47,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/e28663bb8b3947d1ae85405b2c2ef941 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941 2024-11-13T03:52:47,930 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T03:52:47,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/9ae6f7eb33144cd98151b69566da7165 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165 2024-11-13T03:52:47,933 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T03:52:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/29b8d6d144494e27a863957bb0dd1d00 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00 2024-11-13T03:52:47,937 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T03:52:47,937 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 489c6f15a08a8180330774d57ac110a8 in 1243ms, sequenceid=279, compaction requested=true 2024-11-13T03:52:47,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:47,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:47,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T03:52:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T03:52:47,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T03:52:47,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-13T03:52:47,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5530 sec 2024-11-13T03:52:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:48,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:48,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T03:52:48,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/22c6d3e70e274d9094b3ae6210ddbc1c is 50, key is test_row_0/A:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:48,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742293_1469 (size=17181) 2024-11-13T03:52:48,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470028219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470028220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470028336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470028336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T03:52:48,537 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:48,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T03:52:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:48,539 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:48,540 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:48,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470028547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470028547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/22c6d3e70e274d9094b3ae6210ddbc1c 2024-11-13T03:52:48,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/6ad48afe3994442ca1d3d97c42fad5f3 is 50, key is test_row_0/B:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742294_1470 (size=12301) 2024-11-13T03:52:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:48,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:48,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470028867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470028867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41636 deadline: 1731470028927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41602 deadline: 1731470028927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,935 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,935 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731470028947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:48,954 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_0' in 489c6f15a08a8180330774d57ac110a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:52:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:48,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:48,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:48,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:48,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/6ad48afe3994442ca1d3d97c42fad5f3 2024-11-13T03:52:49,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/86d21d6011b644d2af9683c2b9a30579 is 50, key is test_row_0/C:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:49,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742295_1471 (size=12301) 2024-11-13T03:52:49,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:49,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:49,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470029386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470029386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:49,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/86d21d6011b644d2af9683c2b9a30579 2024-11-13T03:52:49,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/22c6d3e70e274d9094b3ae6210ddbc1c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c 2024-11-13T03:52:49,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c, entries=250, sequenceid=317, filesize=16.8 K 2024-11-13T03:52:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/6ad48afe3994442ca1d3d97c42fad5f3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3 2024-11-13T03:52:49,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T03:52:49,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/86d21d6011b644d2af9683c2b9a30579 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579 2024-11-13T03:52:49,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T03:52:49,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for 489c6f15a08a8180330774d57ac110a8 in 1242ms, sequenceid=317, compaction requested=true 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:49,447 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:49,447 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:49,448 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:49,448 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:49,448 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:49,448 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:49,448 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,448 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,448 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/21be84d39536437b9107ceef38756239, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=55.4 K 2024-11-13T03:52:49,448 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/be7fdd9c30904dbf8340763c6fcfce89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=48.2 K 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting be7fdd9c30904dbf8340763c6fcfce89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964375 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21be84d39536437b9107ceef38756239, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964375 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3f8440ac7e45c88c74516086b3e6a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469964540 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b1f8dc01c94d62b0ee99539b59440d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469964528 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae6f7eb33144cd98151b69566da7165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731469965730 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28663bb8b3947d1ae85405b2c2ef941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731469965730 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c6d3e70e274d9094b3ae6210ddbc1c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:49,449 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad48afe3994442ca1d3d97c42fad5f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:49,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:49,457 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:49,458 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/7935fde10f1943dc9d16b2646d5f1a4b is 50, key is test_row_0/A:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:49,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a is 50, key is test_row_2/A:col10/1731469968219/Put/seqid=0 2024-11-13T03:52:49,461 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#403 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:49,461 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/2e85984f75e94964871469ec0b608011 is 50, key is test_row_0/B:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:49,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742296_1472 (size=7415) 2024-11-13T03:52:49,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742298_1474 (size=12779) 2024-11-13T03:52:49,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742297_1473 (size=12779) 2024-11-13T03:52:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:49,871 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a 2024-11-13T03:52:49,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/a2943fccd64c462b9e2b23d086900c60 is 50, key is test_row_2/B:col10/1731469968219/Put/seqid=0 2024-11-13T03:52:49,878 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/7935fde10f1943dc9d16b2646d5f1a4b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/7935fde10f1943dc9d16b2646d5f1a4b 2024-11-13T03:52:49,879 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/2e85984f75e94964871469ec0b608011 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/2e85984f75e94964871469ec0b608011 2024-11-13T03:52:49,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into 2e85984f75e94964871469ec0b608011(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:49,882 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into 7935fde10f1943dc9d16b2646d5f1a4b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:49,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:49,882 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:49,882 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=12, startTime=1731469969447; duration=0sec 2024-11-13T03:52:49,882 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=12, startTime=1731469969447; duration=0sec 2024-11-13T03:52:49,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:49,882 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:49,882 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:49,883 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:49,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742299_1475 (size=7415) 2024-11-13T03:52:49,883 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:52:49,883 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/a2943fccd64c462b9e2b23d086900c60 2024-11-13T03:52:49,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:52:49,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:49,884 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:49,884 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/1861c4068c1543c79b643eb4a4cac53d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=48.2 K 2024-11-13T03:52:49,884 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1861c4068c1543c79b643eb4a4cac53d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731469964375 2024-11-13T03:52:49,885 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 3633cf6820b247cb898d06a5a84cbde8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731469964540 2024-11-13T03:52:49,885 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b8d6d144494e27a863957bb0dd1d00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731469965730 2024-11-13T03:52:49,886 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d21d6011b644d2af9683c2b9a30579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/8e496afa83b745e7ba41cbcb2323400e is 50, key is test_row_2/C:col10/1731469968219/Put/seqid=0 2024-11-13T03:52:49,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742300_1476 (size=7415) 2024-11-13T03:52:49,902 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:49,902 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/4bf859a98b904b358412ce7680dfd2a2 is 50, key is test_row_0/C:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742301_1477 (size=12779) 2024-11-13T03:52:50,300 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/8e496afa83b745e7ba41cbcb2323400e 2024-11-13T03:52:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a 2024-11-13T03:52:50,306 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a, entries=50, sequenceid=324, filesize=7.2 K 2024-11-13T03:52:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/a2943fccd64c462b9e2b23d086900c60 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60 2024-11-13T03:52:50,313 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60, entries=50, sequenceid=324, filesize=7.2 K 2024-11-13T03:52:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/8e496afa83b745e7ba41cbcb2323400e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e 2024-11-13T03:52:50,314 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/4bf859a98b904b358412ce7680dfd2a2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/4bf859a98b904b358412ce7680dfd2a2 2024-11-13T03:52:50,317 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e, entries=50, sequenceid=324, filesize=7.2 K 2024-11-13T03:52:50,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into 4bf859a98b904b358412ce7680dfd2a2(size=12.5 K), total size for store is 19.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:50,317 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:50,317 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=12, startTime=1731469969447; duration=0sec 2024-11-13T03:52:50,318 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:50,318 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:50,318 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 489c6f15a08a8180330774d57ac110a8 in 863ms, sequenceid=324, compaction requested=false 2024-11-13T03:52:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T03:52:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T03:52:50,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T03:52:50,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-11-13T03:52:50,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7830 sec 2024-11-13T03:52:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:50,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T03:52:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:50,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/3dd70b7e9ad442c9acc88bfe244541b0 is 50, key is test_row_1/A:col10/1731469970410/Put/seqid=0 2024-11-13T03:52:50,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742302_1478 (size=14737) 2024-11-13T03:52:50,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469970452, max timestamp=1731469970469Num uniques 859; 2024-11-13T03:52:50,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:50,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731469970452, max timestamp=1731469970469Num uniques 859; 2024-11-13T03:52:50,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T03:52:50,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470030502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470030508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470030617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470030617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T03:52:50,687 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:50,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T03:52:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:50,688 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:50,689 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:50,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:50,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/3dd70b7e9ad442c9acc88bfe244541b0 2024-11-13T03:52:50,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/581dc9541fcf4f649c02d74d7ff2a406 is 50, key is test_row_1/B:col10/1731469970410/Put/seqid=0 2024-11-13T03:52:50,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742303_1479 (size=9857) 2024-11-13T03:52:50,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T03:52:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:50,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:50,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470030837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470030837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:50,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T03:52:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:50,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:51,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T03:52:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:51,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470031156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470031156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/581dc9541fcf4f649c02d74d7ff2a406 2024-11-13T03:52:51,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d403e26c68184bd5882ae23d430d6dc6 is 50, key is test_row_1/C:col10/1731469970410/Put/seqid=0 2024-11-13T03:52:51,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742304_1480 (size=9857) 2024-11-13T03:52:51,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d403e26c68184bd5882ae23d430d6dc6 2024-11-13T03:52:51,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/3dd70b7e9ad442c9acc88bfe244541b0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0 2024-11-13T03:52:51,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0, entries=200, sequenceid=337, filesize=14.4 K 2024-11-13T03:52:51,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/581dc9541fcf4f649c02d74d7ff2a406 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406 2024-11-13T03:52:51,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406, entries=100, sequenceid=337, filesize=9.6 K 2024-11-13T03:52:51,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/d403e26c68184bd5882ae23d430d6dc6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6 2024-11-13T03:52:51,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6, entries=100, sequenceid=337, filesize=9.6 K 2024-11-13T03:52:51,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 489c6f15a08a8180330774d57ac110a8 in 841ms, sequenceid=337, compaction requested=true 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:51,261 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:51,261 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:51,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:51,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:51,262 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:51,262 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:51,262 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:51,262 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,262 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,262 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/7935fde10f1943dc9d16b2646d5f1a4b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=34.1 K 2024-11-13T03:52:51,262 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/2e85984f75e94964871469ec0b608011, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=29.3 K 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7935fde10f1943dc9d16b2646d5f1a4b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e85984f75e94964871469ec0b608011, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a2943fccd64c462b9e2b23d086900c60, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731469968219 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c6446ffc0c4d5e9a6aabe5c8f55b2a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731469968219 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 581dc9541fcf4f649c02d74d7ff2a406, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469970410 2024-11-13T03:52:51,263 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd70b7e9ad442c9acc88bfe244541b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469970407 2024-11-13T03:52:51,268 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:51,268 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/defb7c15e540452392756c4afa631c24 is 50, key is test_row_0/A:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:51,269 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:51,269 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/16646f76e93943cbb1e9d041ab544614 is 50, key is test_row_0/B:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:51,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742305_1481 (size=12881) 2024-11-13T03:52:51,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742306_1482 (size=12881) 2024-11-13T03:52:51,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T03:52:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,298 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/5b778792c38241b2b6180d202da345d9 is 50, key is test_row_0/A:col10/1731469970506/Put/seqid=0 2024-11-13T03:52:51,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742307_1483 (size=12301) 2024-11-13T03:52:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:51,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470031688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470031693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,709 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/5b778792c38241b2b6180d202da345d9 2024-11-13T03:52:51,712 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/defb7c15e540452392756c4afa631c24 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/defb7c15e540452392756c4afa631c24 2024-11-13T03:52:51,713 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/16646f76e93943cbb1e9d041ab544614 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/16646f76e93943cbb1e9d041ab544614 2024-11-13T03:52:51,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/304d4dfb05424e7294b5dfae5450e120 is 50, key is test_row_0/B:col10/1731469970506/Put/seqid=0 2024-11-13T03:52:51,719 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into 16646f76e93943cbb1e9d041ab544614(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:51,719 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:51,719 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=13, startTime=1731469971261; duration=0sec 2024-11-13T03:52:51,719 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:51,719 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:51,719 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:51,720 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into defb7c15e540452392756c4afa631c24(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:51,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:51,720 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=13, startTime=1731469971261; duration=0sec 2024-11-13T03:52:51,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:51,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:51,721 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:51,721 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:51,721 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:51,721 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/4bf859a98b904b358412ce7680dfd2a2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=29.3 K 2024-11-13T03:52:51,721 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf859a98b904b358412ce7680dfd2a2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731469966999 2024-11-13T03:52:51,721 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e496afa83b745e7ba41cbcb2323400e, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731469968219 2024-11-13T03:52:51,722 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting d403e26c68184bd5882ae23d430d6dc6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469970410 2024-11-13T03:52:51,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742308_1484 (size=12301) 2024-11-13T03:52:51,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/304d4dfb05424e7294b5dfae5450e120 2024-11-13T03:52:51,728 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:51,729 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/e22723e9aee84e22913af4c95734a3b0 is 50, key is test_row_0/C:col10/1731469966999/Put/seqid=0 2024-11-13T03:52:51,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742309_1485 (size=12881) 2024-11-13T03:52:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/bcde9276a819480fb1e46929ae9384d0 is 50, key is test_row_0/C:col10/1731469970506/Put/seqid=0 2024-11-13T03:52:51,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742310_1486 (size=12301) 2024-11-13T03:52:51,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470031797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470031807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470032006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470032027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,144 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/e22723e9aee84e22913af4c95734a3b0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e22723e9aee84e22913af4c95734a3b0 2024-11-13T03:52:52,147 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into e22723e9aee84e22913af4c95734a3b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:52,147 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:52,147 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=13, startTime=1731469971261; duration=0sec 2024-11-13T03:52:52,147 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:52,147 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:52,155 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/bcde9276a819480fb1e46929ae9384d0 2024-11-13T03:52:52,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/5b778792c38241b2b6180d202da345d9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9 2024-11-13T03:52:52,161 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9, entries=150, sequenceid=369, filesize=12.0 K 2024-11-13T03:52:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/304d4dfb05424e7294b5dfae5450e120 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120 2024-11-13T03:52:52,165 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120, entries=150, sequenceid=369, filesize=12.0 K 2024-11-13T03:52:52,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/bcde9276a819480fb1e46929ae9384d0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0 2024-11-13T03:52:52,168 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0, entries=150, sequenceid=369, filesize=12.0 K 2024-11-13T03:52:52,169 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=46.96 KB/48090 for 489c6f15a08a8180330774d57ac110a8 in 871ms, sequenceid=369, compaction requested=false 2024-11-13T03:52:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T03:52:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T03:52:52,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T03:52:52,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4820 sec 2024-11-13T03:52:52,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4850 sec 2024-11-13T03:52:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:52,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:52,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/59605989471b4f778d3e802e1597abc4 is 50, key is test_row_0/A:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:52,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742311_1487 (size=14741) 2024-11-13T03:52:52,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469972398, max timestamp=1731469972422Num uniques 859; 2024-11-13T03:52:52,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:52,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469972398, max timestamp=1731469972422Num uniques 859; 2024-11-13T03:52:52,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469972398, max timestamp=1731469972422Num uniques 859; 2024-11-13T03:52:52,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:52,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:52,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470032482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470032483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470032597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470032597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/59605989471b4f778d3e802e1597abc4 2024-11-13T03:52:52,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7db9f70cc76a47f880c97ebb8d71795d is 50, key is test_row_0/B:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:52,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742312_1488 (size=12301) 2024-11-13T03:52:52,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470032807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470032807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T03:52:52,836 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:52,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-13T03:52:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:52,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:52,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:52,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:52,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T03:52:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:52,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470033116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470033116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T03:52:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7db9f70cc76a47f880c97ebb8d71795d 2024-11-13T03:52:53,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/434374974c9049a9a51d14643296e973 is 50, key is test_row_0/C:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:53,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742313_1489 (size=12301) 2024-11-13T03:52:53,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T03:52:53,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T03:52:53,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:53,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:52:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:53,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/434374974c9049a9a51d14643296e973 2024-11-13T03:52:53,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/59605989471b4f778d3e802e1597abc4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4 2024-11-13T03:52:53,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4, entries=200, sequenceid=383, filesize=14.4 K 2024-11-13T03:52:53,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/7db9f70cc76a47f880c97ebb8d71795d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d 2024-11-13T03:52:53,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d, entries=150, sequenceid=383, filesize=12.0 K 2024-11-13T03:52:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/434374974c9049a9a51d14643296e973 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973 2024-11-13T03:52:53,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973, entries=150, sequenceid=383, filesize=12.0 K 2024-11-13T03:52:53,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 489c6f15a08a8180330774d57ac110a8 in 1244ms, sequenceid=383, compaction requested=true 2024-11-13T03:52:53,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:53,577 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:53,577 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:52:53,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:53,577 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:53,577 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/A is initiating minor compaction (all files) 2024-11-13T03:52:53,578 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/A in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,578 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/defb7c15e540452392756c4afa631c24, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=39.0 K 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/B is initiating minor compaction (all files) 2024-11-13T03:52:53,579 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/B in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,579 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/16646f76e93943cbb1e9d041ab544614, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=36.6 K 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting defb7c15e540452392756c4afa631c24, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469966999 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 16646f76e93943cbb1e9d041ab544614, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469966999 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b778792c38241b2b6180d202da345d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731469970488 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 304d4dfb05424e7294b5dfae5450e120, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731469970488 2024-11-13T03:52:53,579 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59605989471b4f778d3e802e1597abc4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731469971676 2024-11-13T03:52:53,580 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db9f70cc76a47f880c97ebb8d71795d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731469971676 2024-11-13T03:52:53,586 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:53,586 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/38373bf4d2d94408a8055985e2477e57 is 50, key is test_row_0/B:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:53,586 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:53,587 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/3bd27ba5283a4128b4f3dd68a80af6aa is 50, key is test_row_0/A:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742314_1490 (size=12983) 2024-11-13T03:52:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742315_1491 (size=12983) 2024-11-13T03:52:53,595 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/3bd27ba5283a4128b4f3dd68a80af6aa as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3bd27ba5283a4128b4f3dd68a80af6aa 2024-11-13T03:52:53,595 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/38373bf4d2d94408a8055985e2477e57 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/38373bf4d2d94408a8055985e2477e57 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/A of 489c6f15a08a8180330774d57ac110a8 into 3bd27ba5283a4128b4f3dd68a80af6aa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/A, priority=13, startTime=1731469973577; duration=0sec 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/B of 489c6f15a08a8180330774d57ac110a8 into 38373bf4d2d94408a8055985e2477e57(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/B, priority=13, startTime=1731469973577; duration=0sec 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:52:53,599 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): 489c6f15a08a8180330774d57ac110a8/C is initiating minor compaction (all files) 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 489c6f15a08a8180330774d57ac110a8/C in TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,599 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e22723e9aee84e22913af4c95734a3b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp, totalSize=36.6 K 2024-11-13T03:52:53,600 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22723e9aee84e22913af4c95734a3b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731469966999 2024-11-13T03:52:53,600 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcde9276a819480fb1e46929ae9384d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731469970488 2024-11-13T03:52:53,600 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434374974c9049a9a51d14643296e973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731469971676 2024-11-13T03:52:53,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:53,602 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:53,606 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 489c6f15a08a8180330774d57ac110a8#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:52:53,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/47c81bab95a74a2c901e6193388227d0 is 50, key is test_row_0/A:col10/1731469972461/Put/seqid=0 2024-11-13T03:52:53,606 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/5b916d5a51b24dae8fa1c3e2b7d1b928 is 50, key is test_row_0/C:col10/1731469972331/Put/seqid=0 2024-11-13T03:52:53,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742317_1493 (size=12983) 2024-11-13T03:52:53,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742316_1492 (size=12301) 2024-11-13T03:52:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:53,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:53,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470033662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470033662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470033777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470033777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470033986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470033997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,016 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/47c81bab95a74a2c901e6193388227d0 2024-11-13T03:52:54,019 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/5b916d5a51b24dae8fa1c3e2b7d1b928 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5b916d5a51b24dae8fa1c3e2b7d1b928 2024-11-13T03:52:54,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/03d64d906ab542dca038263725a5ea33 is 50, key is test_row_0/B:col10/1731469972461/Put/seqid=0 2024-11-13T03:52:54,022 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 489c6f15a08a8180330774d57ac110a8/C of 489c6f15a08a8180330774d57ac110a8 into 5b916d5a51b24dae8fa1c3e2b7d1b928(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:52:54,022 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:54,022 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., storeName=489c6f15a08a8180330774d57ac110a8/C, priority=13, startTime=1731469973577; duration=0sec 2024-11-13T03:52:54,023 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:52:54,023 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:52:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742318_1494 (size=12301) 2024-11-13T03:52:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470034307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470034307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,425 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/03d64d906ab542dca038263725a5ea33 2024-11-13T03:52:54,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/7708d2b86ff5485b8ff5f87d7034bcf0 is 50, key is test_row_0/C:col10/1731469972461/Put/seqid=0 2024-11-13T03:52:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742319_1495 (size=12301) 2024-11-13T03:52:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41650 deadline: 1731470034827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:52:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1731470034827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 2024-11-13T03:52:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 , the old value is region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:52:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8., hostname=ac1a51731247,42123,1731469817167, seqNum=2 because the exception is null or not the one we care about 2024-11-13T03:52:54,834 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/7708d2b86ff5485b8ff5f87d7034bcf0 2024-11-13T03:52:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/47c81bab95a74a2c901e6193388227d0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/47c81bab95a74a2c901e6193388227d0 2024-11-13T03:52:54,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/47c81bab95a74a2c901e6193388227d0, entries=150, sequenceid=420, filesize=12.0 K 2024-11-13T03:52:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/03d64d906ab542dca038263725a5ea33 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/03d64d906ab542dca038263725a5ea33 2024-11-13T03:52:54,842 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/03d64d906ab542dca038263725a5ea33, entries=150, sequenceid=420, filesize=12.0 K 2024-11-13T03:52:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/7708d2b86ff5485b8ff5f87d7034bcf0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/7708d2b86ff5485b8ff5f87d7034bcf0 2024-11-13T03:52:54,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/7708d2b86ff5485b8ff5f87d7034bcf0, entries=150, sequenceid=420, filesize=12.0 K 2024-11-13T03:52:54,846 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 489c6f15a08a8180330774d57ac110a8 in 1244ms, sequenceid=420, compaction requested=false 2024-11-13T03:52:54,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:54,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:54,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-13T03:52:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-13T03:52:54,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T03:52:54,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0090 sec 2024-11-13T03:52:54,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0130 sec 2024-11-13T03:52:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T03:52:54,987 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:54,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:52:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-13T03:52:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:54,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:52:54,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:52:54,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:52:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:55,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T03:52:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:55,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:52:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:52:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/10b19a3db50446dfa6a850704d014ac3 is 50, key is test_row_0/A:col10/1731469973660/Put/seqid=0 2024-11-13T03:52:55,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742320_1496 (size=9857) 2024-11-13T03:52:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:55,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/10b19a3db50446dfa6a850704d014ac3 2024-11-13T03:52:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/63b2a71a7b5e468faf1812faa6776e41 is 50, key is test_row_0/B:col10/1731469973660/Put/seqid=0 2024-11-13T03:52:55,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742321_1497 (size=9857) 2024-11-13T03:52:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:55,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. as already flushing 2024-11-13T03:52:55,895 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-13T03:52:55,895 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,895 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,895 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,895 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-11-13T03:52:55,896 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,896 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,896 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,896 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-11-13T03:52:55,896 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,896 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,896 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,898 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-11-13T03:52:55,898 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,898 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469975874, max timestamp=1731469975895Num uniques 859; 2024-11-13T03:52:55,898 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469975874, max timestamp=1731469975895Num uniques 859; 2024-11-13T03:52:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:55,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731469975874, max timestamp=1731469975895Num uniques 859; 2024-11-13T03:52:55,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T03:52:55,904 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-13T03:52:55,904 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,904 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,904 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,907 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-13T03:52:55,907 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,907 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,907 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,908 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-11-13T03:52:55,908 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:55,908 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,908 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:55,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:55,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/63b2a71a7b5e468faf1812faa6776e41 2024-11-13T03:52:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/6deed190c4734283be2ea1aec355a37c is 50, key is test_row_0/C:col10/1731469973660/Put/seqid=0 2024-11-13T03:52:55,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742322_1498 (size=9857) 2024-11-13T03:52:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:56,374 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/6deed190c4734283be2ea1aec355a37c 2024-11-13T03:52:56,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/10b19a3db50446dfa6a850704d014ac3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/10b19a3db50446dfa6a850704d014ac3 2024-11-13T03:52:56,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/10b19a3db50446dfa6a850704d014ac3, entries=100, sequenceid=432, filesize=9.6 K 2024-11-13T03:52:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/63b2a71a7b5e468faf1812faa6776e41 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/63b2a71a7b5e468faf1812faa6776e41 2024-11-13T03:52:56,392 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/63b2a71a7b5e468faf1812faa6776e41, entries=100, sequenceid=432, filesize=9.6 K 2024-11-13T03:52:56,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/6deed190c4734283be2ea1aec355a37c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/6deed190c4734283be2ea1aec355a37c 2024-11-13T03:52:56,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/6deed190c4734283be2ea1aec355a37c, entries=100, sequenceid=432, filesize=9.6 K 2024-11-13T03:52:56,398 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for 489c6f15a08a8180330774d57ac110a8 in 1255ms, sequenceid=432, compaction requested=true 2024-11-13T03:52:56,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:52:56,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:56,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-13T03:52:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-13T03:52:56,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T03:52:56,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-13T03:52:56,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4130 sec 2024-11-13T03:52:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T03:52:57,139 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:52:58,982 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-13T03:52:58,982 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:58,982 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:58,983 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:58,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:59,028 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-11-13T03:52:59,028 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:59,029 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,029 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:59,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 489c6f15a08a8180330774d57ac110a8 3/3 column families, dataSize=53.67 KB heapSize=141.23 KB 2024-11-13T03:52:59,048 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=A 2024-11-13T03:52:59,049 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:52:59,049 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=B 2024-11-13T03:52:59,049 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 489c6f15a08a8180330774d57ac110a8, store=C 2024-11-13T03:52:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 168 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1268 2024-11-13T03:52:59,049 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3804 rows 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1264 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3792 rows 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1264 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3790 rows 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1262 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3786 rows 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1271 2024-11-13T03:52:59,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3808 rows 2024-11-13T03:52:59,050 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:52:59,050 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:52:59,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:52:59,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:52:59,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:52:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:52:59,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/d4ae210423334e148f4ad3ade5cf7269 is 50, key is test_row_0/A:col10/1731469979047/Put/seqid=0 2024-11-13T03:52:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:52:59,056 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469979056"}]},"ts":"1731469979056"} 2024-11-13T03:52:59,057 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:52:59,057 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:52:59,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:52:59,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742323_1499 (size=12301) 2024-11-13T03:52:59,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, UNASSIGN}] 2024-11-13T03:52:59,061 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, UNASSIGN 2024-11-13T03:52:59,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=489c6f15a08a8180330774d57ac110a8, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:52:59,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, UNASSIGN because future has completed 2024-11-13T03:52:59,063 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:52:59,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:52:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:52:59,216 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:52:59,216 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:52:59,216 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 489c6f15a08a8180330774d57ac110a8, disabling compactions & flushes 2024-11-13T03:52:59,216 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:52:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:52:59,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/d4ae210423334e148f4ad3ade5cf7269 2024-11-13T03:52:59,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/96a4c0d521ea490999c5af725ba8c626 is 50, key is test_row_0/B:col10/1731469979047/Put/seqid=0 2024-11-13T03:52:59,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742324_1500 (size=12301) 2024-11-13T03:52:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:52:59,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/96a4c0d521ea490999c5af725ba8c626 2024-11-13T03:52:59,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/f47fc210b00445b8856038b4f80640b7 is 50, key is test_row_0/C:col10/1731469979047/Put/seqid=0 2024-11-13T03:52:59,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742325_1501 (size=12301) 2024-11-13T03:53:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:53:00,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/f47fc210b00445b8856038b4f80640b7 2024-11-13T03:53:00,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/A/d4ae210423334e148f4ad3ade5cf7269 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/d4ae210423334e148f4ad3ade5cf7269 2024-11-13T03:53:00,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/d4ae210423334e148f4ad3ade5cf7269, entries=150, sequenceid=456, filesize=12.0 K 2024-11-13T03:53:00,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/B/96a4c0d521ea490999c5af725ba8c626 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/96a4c0d521ea490999c5af725ba8c626 2024-11-13T03:53:00,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/96a4c0d521ea490999c5af725ba8c626, entries=150, sequenceid=456, filesize=12.0 K 2024-11-13T03:53:00,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/.tmp/C/f47fc210b00445b8856038b4f80640b7 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/f47fc210b00445b8856038b4f80640b7 2024-11-13T03:53:00,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/f47fc210b00445b8856038b4f80640b7, entries=150, sequenceid=456, filesize=12.0 K 2024-11-13T03:53:00,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.19 KB/144576, currentSize=0 B/0 for 489c6f15a08a8180330774d57ac110a8 in 1274ms, sequenceid=456, compaction requested=true 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 489c6f15a08a8180330774d57ac110a8: 2024-11-13T03:53:00,322 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:00,322 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. because compaction request was cancelled 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:A 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:00,322 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. after waiting 0 ms 2024-11-13T03:53:00,322 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. because compaction request was cancelled 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:B, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:B 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. because compaction request was cancelled 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 489c6f15a08a8180330774d57ac110a8:C, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:00,322 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 489c6f15a08a8180330774d57ac110a8:C 2024-11-13T03:53:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:00,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c650cf1fba444b9fb2cd6f391e937a65, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c498d613b4264ffd9b84151d0116d0cf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/21be84d39536437b9107ceef38756239, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/7935fde10f1943dc9d16b2646d5f1a4b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/defb7c15e540452392756c4afa631c24, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4] to archive 2024-11-13T03:53:00,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:00,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/0a94486eb5da4d7ab6e4ddc22e15f9bc 2024-11-13T03:53:00,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/34b36471750e4ffaa35ff755ae1b0c42 2024-11-13T03:53:00,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/6be6b174a03a48d69d1db11d0a7286c6 2024-11-13T03:53:00,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c650cf1fba444b9fb2cd6f391e937a65 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c650cf1fba444b9fb2cd6f391e937a65 2024-11-13T03:53:00,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/85e94f420c3c4fecab40fe558326ea6f 2024-11-13T03:53:00,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/f3190f6e8af740b4bf26281966b76bae 2024-11-13T03:53:00,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/08fa86101090467299b76038e2d3e2e4 2024-11-13T03:53:00,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c498d613b4264ffd9b84151d0116d0cf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/c498d613b4264ffd9b84151d0116d0cf 2024-11-13T03:53:00,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/11a37af29ecb407fad7932153941f00b 2024-11-13T03:53:00,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/595e435c25ea4aeeac6def397ef3aeb8 2024-11-13T03:53:00,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/21be84d39536437b9107ceef38756239 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/21be84d39536437b9107ceef38756239 2024-11-13T03:53:00,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/b0b1f8dc01c94d62b0ee99539b59440d 2024-11-13T03:53:00,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/e28663bb8b3947d1ae85405b2c2ef941 2024-11-13T03:53:00,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/22c6d3e70e274d9094b3ae6210ddbc1c 2024-11-13T03:53:00,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/7935fde10f1943dc9d16b2646d5f1a4b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/7935fde10f1943dc9d16b2646d5f1a4b 2024-11-13T03:53:00,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/53c6446ffc0c4d5e9a6aabe5c8f55b2a 2024-11-13T03:53:00,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3dd70b7e9ad442c9acc88bfe244541b0 2024-11-13T03:53:00,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/defb7c15e540452392756c4afa631c24 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/defb7c15e540452392756c4afa631c24 2024-11-13T03:53:00,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/5b778792c38241b2b6180d202da345d9 2024-11-13T03:53:00,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/59605989471b4f778d3e802e1597abc4 2024-11-13T03:53:00,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/67cbddbc7caf4721a05de4bd97fc24a8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a28a497c0c084c09911c641fac727ea7, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/be7fdd9c30904dbf8340763c6fcfce89, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/2e85984f75e94964871469ec0b608011, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/16646f76e93943cbb1e9d041ab544614, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d] to archive 2024-11-13T03:53:00,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:00,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7d048dff69c742459a67801d651e62e3 2024-11-13T03:53:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/58fcac5e6253482b8e3945da996404c2 2024-11-13T03:53:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/67cbddbc7caf4721a05de4bd97fc24a8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/67cbddbc7caf4721a05de4bd97fc24a8 2024-11-13T03:53:00,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/daee1df01a74422884230cca03ee286b 2024-11-13T03:53:00,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/162307b5bcf745c79051d4661c2dd959 2024-11-13T03:53:00,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/e47887b8e5b34d90845431751103815a 2024-11-13T03:53:00,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a28a497c0c084c09911c641fac727ea7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a28a497c0c084c09911c641fac727ea7 2024-11-13T03:53:00,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/22240effb367405485565b707509353d 2024-11-13T03:53:00,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/318b83616b2c415ebbd77f0f1dbf2c84 2024-11-13T03:53:00,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/be7fdd9c30904dbf8340763c6fcfce89 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/be7fdd9c30904dbf8340763c6fcfce89 2024-11-13T03:53:00,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/708231201b1e49e1bc4620056094ad5c 2024-11-13T03:53:00,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/0a3f8440ac7e45c88c74516086b3e6a6 2024-11-13T03:53:00,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/9ae6f7eb33144cd98151b69566da7165 2024-11-13T03:53:00,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/2e85984f75e94964871469ec0b608011 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/2e85984f75e94964871469ec0b608011 2024-11-13T03:53:00,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/6ad48afe3994442ca1d3d97c42fad5f3 2024-11-13T03:53:00,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/a2943fccd64c462b9e2b23d086900c60 2024-11-13T03:53:00,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/16646f76e93943cbb1e9d041ab544614 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/16646f76e93943cbb1e9d041ab544614 2024-11-13T03:53:00,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/581dc9541fcf4f649c02d74d7ff2a406 2024-11-13T03:53:00,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/304d4dfb05424e7294b5dfae5450e120 2024-11-13T03:53:00,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/7db9f70cc76a47f880c97ebb8d71795d 2024-11-13T03:53:00,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/ffa16d594606457db3e48aa2ce4446d9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/dfa794c1877f4f60b757e5b6006c6bfd, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/1861c4068c1543c79b643eb4a4cac53d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/4bf859a98b904b358412ce7680dfd2a2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e22723e9aee84e22913af4c95734a3b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973] to archive 2024-11-13T03:53:00,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:00,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/a64fbf098ed243a39e9559d887f27130 2024-11-13T03:53:00,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e6f6e4caee7b40d7abd47bc8ad996b17 2024-11-13T03:53:00,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/ffa16d594606457db3e48aa2ce4446d9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/ffa16d594606457db3e48aa2ce4446d9 2024-11-13T03:53:00,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c8f722f99d014facb1c5afb4cba7538e 2024-11-13T03:53:00,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d8ec1e1c47ee451c91277017d866298b 2024-11-13T03:53:00,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5bbf6e6dc310403d87a1645861dc7128 2024-11-13T03:53:00,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/dfa794c1877f4f60b757e5b6006c6bfd to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/dfa794c1877f4f60b757e5b6006c6bfd 2024-11-13T03:53:00,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/47e4ebc4c00a474b82f07164818aa21a 2024-11-13T03:53:00,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/27fb3286a0684721b6e594228732f097 2024-11-13T03:53:00,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/1861c4068c1543c79b643eb4a4cac53d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/1861c4068c1543c79b643eb4a4cac53d 2024-11-13T03:53:00,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/c925d1ad72f440d388f278178d6f4fc0 2024-11-13T03:53:00,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/3633cf6820b247cb898d06a5a84cbde8 2024-11-13T03:53:00,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/29b8d6d144494e27a863957bb0dd1d00 2024-11-13T03:53:00,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/4bf859a98b904b358412ce7680dfd2a2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/4bf859a98b904b358412ce7680dfd2a2 2024-11-13T03:53:00,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/86d21d6011b644d2af9683c2b9a30579 2024-11-13T03:53:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/8e496afa83b745e7ba41cbcb2323400e 2024-11-13T03:53:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e22723e9aee84e22913af4c95734a3b0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/e22723e9aee84e22913af4c95734a3b0 2024-11-13T03:53:00,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/d403e26c68184bd5882ae23d430d6dc6 2024-11-13T03:53:00,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/bcde9276a819480fb1e46929ae9384d0 2024-11-13T03:53:00,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/434374974c9049a9a51d14643296e973 2024-11-13T03:53:00,376 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/recovered.edits/459.seqid, newMaxSeqId=459, maxSeqId=1 2024-11-13T03:53:00,377 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8. 2024-11-13T03:53:00,377 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 489c6f15a08a8180330774d57ac110a8: Waiting for close lock at 1731469979216Running coprocessor pre-close hooks at 1731469979216Disabling compacts and flushes for region at 1731469979216Disabling writes for close at 1731469980322 (+1106 ms)Writing region close event to WAL at 1731469980373 (+51 ms)Running coprocessor post-close hooks at 1731469980377 (+4 ms)Closed at 1731469980377 2024-11-13T03:53:00,379 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 489c6f15a08a8180330774d57ac110a8 2024-11-13T03:53:00,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=489c6f15a08a8180330774d57ac110a8, regionState=CLOSED 2024-11-13T03:53:00,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:53:00,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T03:53:00,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 489c6f15a08a8180330774d57ac110a8, server=ac1a51731247,42123,1731469817167 in 1.3190 sec 2024-11-13T03:53:00,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T03:53:00,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=489c6f15a08a8180330774d57ac110a8, UNASSIGN in 1.3230 sec 2024-11-13T03:53:00,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-13T03:53:00,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3270 sec 2024-11-13T03:53:00,386 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469980386"}]},"ts":"1731469980386"} 2024-11-13T03:53:00,387 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:53:00,387 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:53:00,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3360 sec 2024-11-13T03:53:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T03:53:01,208 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:01,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:53:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,215 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T03:53:01,217 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,221 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8 2024-11-13T03:53:01,224 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/recovered.edits] 2024-11-13T03:53:01,226 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/10b19a3db50446dfa6a850704d014ac3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/10b19a3db50446dfa6a850704d014ac3 2024-11-13T03:53:01,227 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3bd27ba5283a4128b4f3dd68a80af6aa to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/3bd27ba5283a4128b4f3dd68a80af6aa 2024-11-13T03:53:01,228 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/47c81bab95a74a2c901e6193388227d0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/47c81bab95a74a2c901e6193388227d0 2024-11-13T03:53:01,229 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/d4ae210423334e148f4ad3ade5cf7269 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/A/d4ae210423334e148f4ad3ade5cf7269 2024-11-13T03:53:01,231 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/03d64d906ab542dca038263725a5ea33 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/03d64d906ab542dca038263725a5ea33 2024-11-13T03:53:01,232 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/38373bf4d2d94408a8055985e2477e57 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/38373bf4d2d94408a8055985e2477e57 2024-11-13T03:53:01,233 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/63b2a71a7b5e468faf1812faa6776e41 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/63b2a71a7b5e468faf1812faa6776e41 2024-11-13T03:53:01,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/96a4c0d521ea490999c5af725ba8c626 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/B/96a4c0d521ea490999c5af725ba8c626 2024-11-13T03:53:01,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5b916d5a51b24dae8fa1c3e2b7d1b928 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/5b916d5a51b24dae8fa1c3e2b7d1b928 2024-11-13T03:53:01,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/6deed190c4734283be2ea1aec355a37c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/6deed190c4734283be2ea1aec355a37c 2024-11-13T03:53:01,237 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/7708d2b86ff5485b8ff5f87d7034bcf0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/7708d2b86ff5485b8ff5f87d7034bcf0 2024-11-13T03:53:01,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/f47fc210b00445b8856038b4f80640b7 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/C/f47fc210b00445b8856038b4f80640b7 2024-11-13T03:53:01,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/recovered.edits/459.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8/recovered.edits/459.seqid 2024-11-13T03:53:01,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/489c6f15a08a8180330774d57ac110a8 2024-11-13T03:53:01,240 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:53:01,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,244 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:53:01,245 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:53:01,246 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,246 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:53:01,246 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731469981246"}]},"ts":"9223372036854775807"} 2024-11-13T03:53:01,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:53:01,248 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 489c6f15a08a8180330774d57ac110a8, NAME => 'TestAcidGuarantees,,1731469953685.489c6f15a08a8180330774d57ac110a8.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:53:01,248 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:53:01,248 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731469981248"}]},"ts":"9223372036854775807"} 2024-11-13T03:53:01,250 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:53:01,250 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-13T03:53:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T03:53:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:53:01,329 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:01,336 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 249), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 310), ProcessCount=11 (was 11), AvailableMemoryMB=8124 (was 8145) 2024-11-13T03:53:01,345 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=8124 2024-11-13T03:53:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:53:01,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:53:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:01,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:53:01,348 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:01,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-13T03:53:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:01,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:53:01,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742326_1502 (size=1061) 2024-11-13T03:53:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:01,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e 2024-11-13T03:53:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742327_1503 (size=53) 2024-11-13T03:53:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:02,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:53:02,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d7159452becbf53ae4db2364b5f6792c, disabling compactions & flushes 2024-11-13T03:53:02,171 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. after waiting 0 ms 2024-11-13T03:53:02,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7159452becbf53ae4db2364b5f6792c: Waiting for close lock at 1731469982171Disabling compacts and flushes for region at 1731469982171Disabling writes for close at 1731469982172 (+1 ms)Writing region close event to WAL at 1731469982172Closed at 1731469982172 2024-11-13T03:53:02,175 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:53:02,175 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731469982175"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731469982175"}]},"ts":"1731469982175"} 2024-11-13T03:53:02,179 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:53:02,180 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:53:02,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469982180"}]},"ts":"1731469982180"} 2024-11-13T03:53:02,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T03:53:02,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, ASSIGN}] 2024-11-13T03:53:02,184 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, ASSIGN 2024-11-13T03:53:02,185 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, ASSIGN; state=OFFLINE, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=false 2024-11-13T03:53:02,335 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, ASSIGN because future has completed 2024-11-13T03:53:02,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:53:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:02,497 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,497 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:53:02,498 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,498 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:53:02,498 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,498 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,500 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,503 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:02,503 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName A 2024-11-13T03:53:02,503 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:02,504 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:02,504 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,506 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:02,506 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName B 2024-11-13T03:53:02,506 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:02,507 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:02,507 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,508 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:02,509 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName C 2024-11-13T03:53:02,509 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:02,509 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:02,509 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,510 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,511 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,511 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,513 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,513 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,513 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:53:02,515 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,518 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:53:02,519 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened d7159452becbf53ae4db2364b5f6792c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74262135, jitterRate=0.10659204423427582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:53:02,519 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:02,519 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for d7159452becbf53ae4db2364b5f6792c: Running coprocessor pre-open hook at 1731469982498Writing region info on filesystem at 1731469982498Initializing all the Stores at 1731469982500 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469982500Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469982500Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469982500Cleaning up temporary data from old regions at 1731469982513 (+13 ms)Running coprocessor post-open hooks at 1731469982519 (+6 ms)Region opened successfully at 1731469982519 2024-11-13T03:53:02,520 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., pid=144, masterSystemTime=1731469982489 2024-11-13T03:53:02,521 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,521 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:02,522 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=OPEN, openSeqNum=2, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:02,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:53:02,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T03:53:02,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 in 187 msec 2024-11-13T03:53:02,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T03:53:02,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, ASSIGN in 343 msec 2024-11-13T03:53:02,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:53:02,528 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731469982528"}]},"ts":"1731469982528"} 2024-11-13T03:53:02,529 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T03:53:02,529 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:53:02,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-13T03:53:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T03:53:03,499 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:03,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:03,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:03,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:03,503 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ab5178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:03,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ab227a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:03,508 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:03,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:03,511 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:53:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ac1a51731247,33933,1731469816394 2024-11-13T03:53:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d795b5c 2024-11-13T03:53:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:53:03,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:53:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T03:53:03,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:53:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:03,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742328_1504 (size=1097) 2024-11-13T03:53:03,924 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T03:53:03,924 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T03:53:03,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:53:03,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE}] 2024-11-13T03:53:03,934 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE 2024-11-13T03:53:03,935 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:03,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE because future has completed 2024-11-13T03:53:03,936 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:53:03,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:53:04,089 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,090 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:53:04,090 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing d7159452becbf53ae4db2364b5f6792c, disabling compactions & flushes 2024-11-13T03:53:04,090 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,090 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,090 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. after waiting 0 ms 2024-11-13T03:53:04,090 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,098 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T03:53:04,099 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,099 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for d7159452becbf53ae4db2364b5f6792c: Waiting for close lock at 1731469984090Running coprocessor pre-close hooks at 1731469984090Disabling compacts and flushes for region at 1731469984090Disabling writes for close at 1731469984090Writing region close event to WAL at 1731469984091 (+1 ms)Running coprocessor post-close hooks at 1731469984099 (+8 ms)Closed at 1731469984099 2024-11-13T03:53:04,099 WARN [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: d7159452becbf53ae4db2364b5f6792c to self. 2024-11-13T03:53:04,102 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=CLOSED 2024-11-13T03:53:04,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:53:04,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-13T03:53:04,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 in 172 msec 2024-11-13T03:53:04,113 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE; state=CLOSED, location=ac1a51731247,42123,1731469817167; forceNewPlan=false, retain=true 2024-11-13T03:53:04,264 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=OPENING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE because future has completed 2024-11-13T03:53:04,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:53:04,431 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,431 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:53:04,432 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,432 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:53:04,432 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,432 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,435 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,435 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:04,435 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName A 2024-11-13T03:53:04,437 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:04,437 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:04,437 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,438 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:04,438 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName B 2024-11-13T03:53:04,438 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:04,439 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:04,439 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,439 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T03:53:04,439 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7159452becbf53ae4db2364b5f6792c columnFamilyName C 2024-11-13T03:53:04,440 DEBUG [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:04,440 INFO [StoreOpener-d7159452becbf53ae4db2364b5f6792c-1 {}] regionserver.HStore(327): Store=d7159452becbf53ae4db2364b5f6792c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:53:04,440 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,440 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,441 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,442 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,443 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,443 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,443 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:53:04,444 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,445 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened d7159452becbf53ae4db2364b5f6792c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63184343, jitterRate=-0.0584799200296402}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:53:04,445 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,446 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for d7159452becbf53ae4db2364b5f6792c: Running coprocessor pre-open hook at 1731469984433Writing region info on filesystem at 1731469984433Initializing all the Stores at 1731469984434 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469984434Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469984434Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731469984434Cleaning up temporary data from old regions at 1731469984443 (+9 ms)Running coprocessor post-open hooks at 1731469984445 (+2 ms)Region opened successfully at 1731469984446 (+1 ms) 2024-11-13T03:53:04,447 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., pid=149, masterSystemTime=1731469984424 2024-11-13T03:53:04,448 DEBUG [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,448 INFO [RS_OPEN_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,448 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=OPEN, openSeqNum=5, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:53:04,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-13T03:53:04,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 in 181 msec 2024-11-13T03:53:04,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T03:53:04,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, REOPEN/MOVE in 519 msec 2024-11-13T03:53:04,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-13T03:53:04,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-13T03:53:04,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-13T03:53:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-13T03:53:04,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:04,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edc9e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7844cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,462 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e519caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69834db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676d8722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e953375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26fce659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b298cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,472 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55973322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,473 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7fee3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64852367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,475 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b1525e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df1119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ee4ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c600973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e68cf19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d738352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,484 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b59b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ac1a51731247,33933,-1 for getting cluster id 2024-11-13T03:53:04,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:53:04,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae6c169b-cfc2-463c-960c-933eaa38c2d3' 2024-11-13T03:53:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:53:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae6c169b-cfc2-463c-960c-933eaa38c2d3" 2024-11-13T03:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2a47de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ac1a51731247,33933,-1] 2024-11-13T03:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:04,488 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:53:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a9553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:53:04,489 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,489 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,489 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,490 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,490 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:04,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,491 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,492 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,492 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-13T03:53:04,493 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:04,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:04,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:04,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,498 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:53:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ac1a51731247,42123,1731469817167, seqNum=-1] 2024-11-13T03:53:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:53:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:53:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5] 2024-11-13T03:53:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:04,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:04,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113657eced0b72c43aa9a71492456eb98aa_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469984501/Put/seqid=0 2024-11-13T03:53:04,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469984518, max timestamp=1731469984534Num uniques 859; 2024-11-13T03:53:04,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469984518, max timestamp=1731469984534Num uniques 859; 2024-11-13T03:53:04,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:53:04,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:53:04,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731469984518, max timestamp=1731469984534Num uniques 859; 2024-11-13T03:53:04,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ac1a51731247:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T03:53:04,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742329_1505 (size=12154) 2024-11-13T03:53:04,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:04,552 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113657eced0b72c43aa9a71492456eb98aa_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113657eced0b72c43aa9a71492456eb98aa_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:04,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/0f28e0e421124d78b57f6cc30a7caba0, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:04,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/0f28e0e421124d78b57f6cc30a7caba0 is 175, key is test_row_0/A:col10/1731469984501/Put/seqid=0 2024-11-13T03:53:04,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470044551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470044555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470044555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470044556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470044557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742330_1506 (size=30955) 2024-11-13T03:53:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:04,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T03:53:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470044666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470044667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470044667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470044667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470044677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T03:53:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:04,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470044877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470044886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:04,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T03:53:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:04,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:04,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/0f28e0e421124d78b57f6cc30a7caba0 2024-11-13T03:53:04,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2ac3725c70c94eb1bf201c48240e306b is 50, key is test_row_0/B:col10/1731469984501/Put/seqid=0 2024-11-13T03:53:05,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742331_1507 (size=12001) 2024-11-13T03:53:05,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2ac3725c70c94eb1bf201c48240e306b 2024-11-13T03:53:05,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/ed581c5faa724e22adce59e10e867b9c is 50, key is test_row_0/C:col10/1731469984501/Put/seqid=0 2024-11-13T03:53:05,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742332_1508 (size=12001) 2024-11-13T03:53:05,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/ed581c5faa724e22adce59e10e867b9c 2024-11-13T03:53:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/0f28e0e421124d78b57f6cc30a7caba0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0 2024-11-13T03:53:05,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0, entries=150, sequenceid=15, filesize=30.2 K 2024-11-13T03:53:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2ac3725c70c94eb1bf201c48240e306b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b 2024-11-13T03:53:05,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T03:53:05,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/ed581c5faa724e22adce59e10e867b9c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c 2024-11-13T03:53:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T03:53:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d7159452becbf53ae4db2364b5f6792c in 535ms, sequenceid=15, compaction requested=false 2024-11-13T03:53:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T03:53:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:05,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:05,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139ce2f41488a04853bbbd9869aef46188_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469984555/Put/seqid=0 2024-11-13T03:53:05,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742333_1509 (size=12154) 2024-11-13T03:53:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:05,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:05,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470045191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470045194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470045195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470045196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470045197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470045307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470045307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470045307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470045309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:05,519 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139ce2f41488a04853bbbd9869aef46188_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139ce2f41488a04853bbbd9869aef46188_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470045517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470045517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c7ad8246a28f4d199bc872e2f61af44b, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470045517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c7ad8246a28f4d199bc872e2f61af44b is 175, key is test_row_0/A:col10/1731469984555/Put/seqid=0 2024-11-13T03:53:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470045517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742334_1510 (size=30955) 2024-11-13T03:53:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470045707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470045827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470045827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470045827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470045827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:05,900 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:53:05,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c7ad8246a28f4d199bc872e2f61af44b 2024-11-13T03:53:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/aa42c75584cc48e79cacc4400005e512 is 50, key is test_row_0/B:col10/1731469984555/Put/seqid=0 2024-11-13T03:53:05,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742335_1511 (size=12001) 2024-11-13T03:53:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470046337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:06,339 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/aa42c75584cc48e79cacc4400005e512 2024-11-13T03:53:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470046337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470046337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470046337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fdeeafb67f6640f6abf8541adf17cc6a is 50, key is test_row_0/C:col10/1731469984555/Put/seqid=0 2024-11-13T03:53:06,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742336_1512 (size=12001) 2024-11-13T03:53:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470046717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:06,719 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:06,749 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fdeeafb67f6640f6abf8541adf17cc6a 2024-11-13T03:53:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c7ad8246a28f4d199bc872e2f61af44b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b 2024-11-13T03:53:06,755 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b, entries=150, sequenceid=50, filesize=30.2 K 2024-11-13T03:53:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/aa42c75584cc48e79cacc4400005e512 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512 2024-11-13T03:53:06,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T03:53:06,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fdeeafb67f6640f6abf8541adf17cc6a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a 2024-11-13T03:53:06,761 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T03:53:06,762 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d7159452becbf53ae4db2364b5f6792c in 1654ms, sequenceid=50, compaction requested=false 2024-11-13T03:53:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-13T03:53:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-13T03:53:06,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T03:53:06,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2660 sec 2024-11-13T03:53:06,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2740 sec 2024-11-13T03:53:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:07,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:07,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa29000c849b401689f1672bd49c2521_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742337_1513 (size=14594) 2024-11-13T03:53:07,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470047371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470047376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470047377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470047377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470047487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470047487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470047487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470047487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470047697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470047697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470047697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470047697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:07,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:07,812 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa29000c849b401689f1672bd49c2521_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa29000c849b401689f1672bd49c2521_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:07,813 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/f433b9e2ad4740be8a360e893ce61933, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:07,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/f433b9e2ad4740be8a360e893ce61933 is 175, key is test_row_0/A:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:07,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742338_1514 (size=39549) 2024-11-13T03:53:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470048006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470048006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470048006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470048007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,217 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/f433b9e2ad4740be8a360e893ce61933 2024-11-13T03:53:08,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/f5bae0bafbc549afb9492c0a5ff48ebc is 50, key is test_row_0/B:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:08,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742339_1515 (size=12001) 2024-11-13T03:53:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470048516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470048516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470048517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470048517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/f5bae0bafbc549afb9492c0a5ff48ebc 2024-11-13T03:53:08,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/07e155bed6cc487b902a83610906b1af is 50, key is test_row_0/C:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:08,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742340_1516 (size=12001) 2024-11-13T03:53:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T03:53:08,657 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:08,657 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T03:53:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:08,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:08,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:08,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470048726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:08,728 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:08,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T03:53:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:08,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T03:53:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:08,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:09,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/07e155bed6cc487b902a83610906b1af 2024-11-13T03:53:09,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/f433b9e2ad4740be8a360e893ce61933 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933 2024-11-13T03:53:09,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933, entries=200, sequenceid=63, filesize=38.6 K 2024-11-13T03:53:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/f5bae0bafbc549afb9492c0a5ff48ebc as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc 2024-11-13T03:53:09,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T03:53:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/07e155bed6cc487b902a83610906b1af as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af 2024-11-13T03:53:09,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T03:53:09,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d7159452becbf53ae4db2364b5f6792c in 1708ms, sequenceid=63, compaction requested=true 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:09,057 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:09,057 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:53:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/B is initiating minor compaction (all files) 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/A is initiating minor compaction (all files) 2024-11-13T03:53:09,058 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/B in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:09,058 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/A in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:09,058 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.2 K 2024-11-13T03:53:09,058 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=99.1 K 2024-11-13T03:53:09,058 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933] 2024-11-13T03:53:09,058 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac3725c70c94eb1bf201c48240e306b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469984501 2024-11-13T03:53:09,059 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f28e0e421124d78b57f6cc30a7caba0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469984501 2024-11-13T03:53:09,059 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting aa42c75584cc48e79cacc4400005e512, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731469984553 2024-11-13T03:53:09,059 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ad8246a28f4d199bc872e2f61af44b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731469984553 2024-11-13T03:53:09,059 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bae0bafbc549afb9492c0a5ff48ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:09,059 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting f433b9e2ad4740be8a360e893ce61933, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:09,065 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:09,065 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:09,065 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/005f7e16df1e4b709b6e90a801dcd9de is 50, key is test_row_0/B:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:09,071 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111372c72ed4e36c48779f1ef4bc12abd0b8_d7159452becbf53ae4db2364b5f6792c store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:09,073 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111372c72ed4e36c48779f1ef4bc12abd0b8_d7159452becbf53ae4db2364b5f6792c, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:09,073 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111372c72ed4e36c48779f1ef4bc12abd0b8_d7159452becbf53ae4db2364b5f6792c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:09,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742341_1517 (size=12104) 2024-11-13T03:53:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742342_1518 (size=4469) 2024-11-13T03:53:09,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:09,117 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:09,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113835fd5ebcdb046d8a397a7e0c72022c3_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469987375/Put/seqid=0 2024-11-13T03:53:09,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742343_1519 (size=12154) 2024-11-13T03:53:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:09,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113835fd5ebcdb046d8a397a7e0c72022c3_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113835fd5ebcdb046d8a397a7e0c72022c3_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/07a62a369ba2406b81472a53a6e50789, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/07a62a369ba2406b81472a53a6e50789 is 175, key is test_row_0/A:col10/1731469987375/Put/seqid=0 2024-11-13T03:53:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742344_1520 (size=30955) 2024-11-13T03:53:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:09,478 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/005f7e16df1e4b709b6e90a801dcd9de as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/005f7e16df1e4b709b6e90a801dcd9de 2024-11-13T03:53:09,482 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/B of d7159452becbf53ae4db2364b5f6792c into 005f7e16df1e4b709b6e90a801dcd9de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:09,482 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:09,482 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/B, priority=13, startTime=1731469989057; duration=0sec 2024-11-13T03:53:09,482 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:09,482 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:B 2024-11-13T03:53:09,482 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:09,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:09,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/C is initiating minor compaction (all files) 2024-11-13T03:53:09,483 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/C in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:09,483 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.2 K 2024-11-13T03:53:09,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ed581c5faa724e22adce59e10e867b9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731469984501 2024-11-13T03:53:09,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting fdeeafb67f6640f6abf8541adf17cc6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731469984553 2024-11-13T03:53:09,483 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e155bed6cc487b902a83610906b1af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:09,485 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#A#compaction#442 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:09,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/de557ba49db24d04b7c177afb2efb64b is 175, key is test_row_0/A:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:09,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742345_1521 (size=31058) 2024-11-13T03:53:09,489 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:09,489 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/e20a46d3f2cc4da5a2bb284c9a589dec is 50, key is test_row_0/C:col10/1731469987348/Put/seqid=0 2024-11-13T03:53:09,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742346_1522 (size=12104) 2024-11-13T03:53:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:09,539 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/07a62a369ba2406b81472a53a6e50789 2024-11-13T03:53:09,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470049539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/67154ea5a32248449d79405cc717ae00 is 50, key is test_row_0/B:col10/1731469987375/Put/seqid=0 2024-11-13T03:53:09,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470049542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742347_1523 (size=12001) 2024-11-13T03:53:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470049544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470049544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470049647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470049656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470049656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470049657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:09,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470049857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470049866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470049866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470049867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:09,893 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/de557ba49db24d04b7c177afb2efb64b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b 2024-11-13T03:53:09,897 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/e20a46d3f2cc4da5a2bb284c9a589dec as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e20a46d3f2cc4da5a2bb284c9a589dec 2024-11-13T03:53:09,898 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/A of d7159452becbf53ae4db2364b5f6792c into de557ba49db24d04b7c177afb2efb64b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:09,898 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:09,898 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/A, priority=13, startTime=1731469989057; duration=0sec 2024-11-13T03:53:09,898 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:09,898 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:A 2024-11-13T03:53:09,906 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/C of d7159452becbf53ae4db2364b5f6792c into e20a46d3f2cc4da5a2bb284c9a589dec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:09,906 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:09,906 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/C, priority=13, startTime=1731469989057; duration=0sec 2024-11-13T03:53:09,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:09,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:C 2024-11-13T03:53:09,948 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/67154ea5a32248449d79405cc717ae00 2024-11-13T03:53:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1ebe6c5665244fd986e26acd94c64cf2 is 50, key is test_row_0/C:col10/1731469987375/Put/seqid=0 2024-11-13T03:53:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742348_1524 (size=12001) 2024-11-13T03:53:10,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470050167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470050177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470050177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470050177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,360 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1ebe6c5665244fd986e26acd94c64cf2 2024-11-13T03:53:10,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/07a62a369ba2406b81472a53a6e50789 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789 2024-11-13T03:53:10,366 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789, entries=150, sequenceid=83, filesize=30.2 K 2024-11-13T03:53:10,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/67154ea5a32248449d79405cc717ae00 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00 2024-11-13T03:53:10,369 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00, entries=150, sequenceid=83, filesize=11.7 K 2024-11-13T03:53:10,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1ebe6c5665244fd986e26acd94c64cf2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2 2024-11-13T03:53:10,372 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2, entries=150, sequenceid=83, filesize=11.7 K 2024-11-13T03:53:10,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for d7159452becbf53ae4db2364b5f6792c in 1256ms, sequenceid=83, compaction requested=false 2024-11-13T03:53:10,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:10,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:10,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T03:53:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T03:53:10,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T03:53:10,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-13T03:53:10,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7200 sec 2024-11-13T03:53:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:10,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:10,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d364f5ae5e5491fab47687dfd3c7a74_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:10,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742349_1525 (size=14594) 2024-11-13T03:53:10,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470050703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470050704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470050705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470050705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T03:53:10,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:10,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T03:53:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:10,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:10,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:10,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:10,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470050817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470050817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470050817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470050817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:10,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:10,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470051027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470051027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470051027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470051027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:11,091 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d364f5ae5e5491fab47687dfd3c7a74_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d364f5ae5e5491fab47687dfd3c7a74_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:11,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/782a59718b2643a4b4c1d368783db181, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:11,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/782a59718b2643a4b4c1d368783db181 is 175, key is test_row_0/A:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:11,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742350_1526 (size=39549) 2024-11-13T03:53:11,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:11,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470051336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470051336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470051337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470051337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:11,495 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/782a59718b2643a4b4c1d368783db181 2024-11-13T03:53:11,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/c94cd0ce90314526a35e839b0cf2fa4e is 50, key is test_row_0/B:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:11,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742351_1527 (size=12001) 2024-11-13T03:53:11,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/c94cd0ce90314526a35e839b0cf2fa4e 2024-11-13T03:53:11,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/b4eecc3c3cf24a6b8cb9021806cabe8d is 50, key is test_row_0/C:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:11,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742352_1528 (size=12001) 2024-11-13T03:53:11,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470051846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470051846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470051847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470051847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:11,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:11,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:11,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:11,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/b4eecc3c3cf24a6b8cb9021806cabe8d 2024-11-13T03:53:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/782a59718b2643a4b4c1d368783db181 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181 2024-11-13T03:53:11,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181, entries=200, sequenceid=100, filesize=38.6 K 2024-11-13T03:53:11,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/c94cd0ce90314526a35e839b0cf2fa4e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e 2024-11-13T03:53:11,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e, entries=150, sequenceid=100, filesize=11.7 K 2024-11-13T03:53:11,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/b4eecc3c3cf24a6b8cb9021806cabe8d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d 2024-11-13T03:53:11,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-13T03:53:11,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d7159452becbf53ae4db2364b5f6792c in 1251ms, sequenceid=100, compaction requested=true 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:11,932 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:11,932 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:53:11,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:11,934 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:11,934 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/A is initiating minor compaction (all files) 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/B is initiating minor compaction (all files) 2024-11-13T03:53:11,935 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/A in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,935 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/B in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,935 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/005f7e16df1e4b709b6e90a801dcd9de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.3 K 2024-11-13T03:53:11,935 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=99.2 K 2024-11-13T03:53:11,935 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181] 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 005f7e16df1e4b709b6e90a801dcd9de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting de557ba49db24d04b7c177afb2efb64b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 67154ea5a32248449d79405cc717ae00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731469987370 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a62a369ba2406b81472a53a6e50789, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731469987370 2024-11-13T03:53:11,935 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c94cd0ce90314526a35e839b0cf2fa4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:11,936 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782a59718b2643a4b4c1d368783db181, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:11,941 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:11,953 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:11,953 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/cf5ad3b54ef0431dbf82653bfdc2c771 is 50, key is test_row_0/B:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:11,971 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138d35d180bb4b410bb12d992e53a6f4da_d7159452becbf53ae4db2364b5f6792c store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:11,973 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138d35d180bb4b410bb12d992e53a6f4da_d7159452becbf53ae4db2364b5f6792c, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:11,974 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138d35d180bb4b410bb12d992e53a6f4da_d7159452becbf53ae4db2364b5f6792c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742353_1529 (size=12207) 2024-11-13T03:53:11,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742354_1530 (size=4469) 2024-11-13T03:53:12,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:12,029 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:12,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130563e9e455034aabb0855f0df672fcee_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469990703/Put/seqid=0 2024-11-13T03:53:12,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742355_1531 (size=12154) 2024-11-13T03:53:12,396 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/cf5ad3b54ef0431dbf82653bfdc2c771 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/cf5ad3b54ef0431dbf82653bfdc2c771 2024-11-13T03:53:12,397 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#A#compaction#450 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:12,398 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/6a1e30edd3b94b658874fa68dae8ddf0 is 175, key is test_row_0/A:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:12,403 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/B of d7159452becbf53ae4db2364b5f6792c into cf5ad3b54ef0431dbf82653bfdc2c771(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:12,403 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:12,403 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/B, priority=13, startTime=1731469991932; duration=0sec 2024-11-13T03:53:12,403 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:12,403 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:B 2024-11-13T03:53:12,403 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:12,404 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:12,404 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/C is initiating minor compaction (all files) 2024-11-13T03:53:12,404 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/C in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:12,405 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e20a46d3f2cc4da5a2bb284c9a589dec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.3 K 2024-11-13T03:53:12,405 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e20a46d3f2cc4da5a2bb284c9a589dec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731469985195 2024-11-13T03:53:12,405 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ebe6c5665244fd986e26acd94c64cf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731469987370 2024-11-13T03:53:12,405 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting b4eecc3c3cf24a6b8cb9021806cabe8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:12,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742356_1532 (size=31161) 2024-11-13T03:53:12,413 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:12,413 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/7e2bf67b67b148ae9716de0912e52b14 is 50, key is test_row_0/C:col10/1731469989541/Put/seqid=0 2024-11-13T03:53:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742357_1533 (size=12207) 2024-11-13T03:53:12,418 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/6a1e30edd3b94b658874fa68dae8ddf0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0 2024-11-13T03:53:12,422 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/A of d7159452becbf53ae4db2364b5f6792c into 6a1e30edd3b94b658874fa68dae8ddf0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:12,422 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:12,422 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/A, priority=13, startTime=1731469991932; duration=0sec 2024-11-13T03:53:12,422 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:12,422 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:A 2024-11-13T03:53:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:12,448 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130563e9e455034aabb0855f0df672fcee_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130563e9e455034aabb0855f0df672fcee_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/079292030f224f95983129e2146d373c, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/079292030f224f95983129e2146d373c is 175, key is test_row_0/A:col10/1731469990703/Put/seqid=0 2024-11-13T03:53:12,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742358_1534 (size=30955) 2024-11-13T03:53:12,453 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/079292030f224f95983129e2146d373c 2024-11-13T03:53:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/306953ad7ba04ff7985ce7a9721a6245 is 50, key is test_row_0/B:col10/1731469990703/Put/seqid=0 2024-11-13T03:53:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742359_1535 (size=12001) 2024-11-13T03:53:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:12,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470052794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,820 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/7e2bf67b67b148ae9716de0912e52b14 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/7e2bf67b67b148ae9716de0912e52b14 2024-11-13T03:53:12,823 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/C of d7159452becbf53ae4db2364b5f6792c into 7e2bf67b67b148ae9716de0912e52b14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:12,823 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:12,823 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/C, priority=13, startTime=1731469991932; duration=0sec 2024-11-13T03:53:12,823 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:12,823 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:C 2024-11-13T03:53:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470052857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470052857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470052857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,860 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,860 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,860 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,864 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/306953ad7ba04ff7985ce7a9721a6245 2024-11-13T03:53:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/903c254eadab411ba1a772febac4f23b is 50, key is test_row_0/C:col10/1731469990703/Put/seqid=0 2024-11-13T03:53:12,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470052867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,870 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742360_1536 (size=12001) 2024-11-13T03:53:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470052906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470053117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:13,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/903c254eadab411ba1a772febac4f23b 2024-11-13T03:53:13,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/079292030f224f95983129e2146d373c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c 2024-11-13T03:53:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c, entries=150, sequenceid=118, filesize=30.2 K 2024-11-13T03:53:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/306953ad7ba04ff7985ce7a9721a6245 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245 2024-11-13T03:53:13,288 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T03:53:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/903c254eadab411ba1a772febac4f23b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b 2024-11-13T03:53:13,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T03:53:13,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d7159452becbf53ae4db2364b5f6792c in 1263ms, sequenceid=118, compaction requested=false 2024-11-13T03:53:13,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:13,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:13,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T03:53:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T03:53:13,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T03:53:13,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4850 sec 2024-11-13T03:53:13,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4890 sec 2024-11-13T03:53:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:13,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:13,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139aa22fd3efa14d60a6eac6769df3c35a_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742361_1537 (size=14744) 2024-11-13T03:53:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470053484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470053597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:13,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470053806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:13,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:13,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139aa22fd3efa14d60a6eac6769df3c35a_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aa22fd3efa14d60a6eac6769df3c35a_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:13,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e11cccef148646a396c585bacd2025cf, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:13,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e11cccef148646a396c585bacd2025cf is 175, key is test_row_0/A:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:13,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742362_1538 (size=39699) 2024-11-13T03:53:13,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e11cccef148646a396c585bacd2025cf 2024-11-13T03:53:13,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/55eae16d33f84bc0aaad9a785d3637d4 is 50, key is test_row_0/B:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:13,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742363_1539 (size=12101) 2024-11-13T03:53:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470054117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/55eae16d33f84bc0aaad9a785d3637d4 2024-11-13T03:53:14,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/e11337fd7f114e5dbf4f9bbc6705e8ad is 50, key is test_row_0/C:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:14,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742364_1540 (size=12101) 2024-11-13T03:53:14,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470054627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/e11337fd7f114e5dbf4f9bbc6705e8ad 2024-11-13T03:53:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e11cccef148646a396c585bacd2025cf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf 2024-11-13T03:53:14,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf, entries=200, sequenceid=136, filesize=38.8 K 2024-11-13T03:53:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/55eae16d33f84bc0aaad9a785d3637d4 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4 2024-11-13T03:53:14,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T03:53:14,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/e11337fd7f114e5dbf4f9bbc6705e8ad as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad 2024-11-13T03:53:14,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T03:53:14,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d7159452becbf53ae4db2364b5f6792c in 1254ms, sequenceid=136, compaction requested=true 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:14,683 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:53:14,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T03:53:14,684 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:14,684 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/A is initiating minor compaction (all files) 2024-11-13T03:53:14,684 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/A in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:14,684 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=99.4 K 2024-11-13T03:53:14,684 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:14,684 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf] 2024-11-13T03:53:14,685 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a1e30edd3b94b658874fa68dae8ddf0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:14,685 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 079292030f224f95983129e2146d373c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731469990703 2024-11-13T03:53:14,685 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting e11cccef148646a396c585bacd2025cf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:14,685 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:14,685 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:14,686 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/B is initiating minor compaction (all files) 2024-11-13T03:53:14,686 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/B in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:14,686 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/cf5ad3b54ef0431dbf82653bfdc2c771, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.5 K 2024-11-13T03:53:14,686 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5ad3b54ef0431dbf82653bfdc2c771, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:14,686 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 306953ad7ba04ff7985ce7a9721a6245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731469990703 2024-11-13T03:53:14,686 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55eae16d33f84bc0aaad9a785d3637d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:14,689 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:14,691 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cd0045116b4f45b1b208aa378ace8a4f_d7159452becbf53ae4db2364b5f6792c store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:14,692 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:14,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cd0045116b4f45b1b208aa378ace8a4f_d7159452becbf53ae4db2364b5f6792c, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:14,693 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cd0045116b4f45b1b208aa378ace8a4f_d7159452becbf53ae4db2364b5f6792c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:14,694 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/5d64901e163349ef90c9eb5f4b06bb7c is 50, key is test_row_0/B:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:14,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742365_1541 (size=4469) 2024-11-13T03:53:14,704 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#A#compaction#459 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:14,704 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/8057755f091f41bdbd40ce86f9fdf2b0 is 175, key is test_row_0/A:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:14,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742366_1542 (size=12409) 2024-11-13T03:53:14,716 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/5d64901e163349ef90c9eb5f4b06bb7c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/5d64901e163349ef90c9eb5f4b06bb7c 2024-11-13T03:53:14,720 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/B of d7159452becbf53ae4db2364b5f6792c into 5d64901e163349ef90c9eb5f4b06bb7c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:14,720 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/B, priority=13, startTime=1731469994683; duration=0sec 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:B 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T03:53:14,720 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/C is initiating minor compaction (all files) 2024-11-13T03:53:14,721 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/C in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:14,721 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/7e2bf67b67b148ae9716de0912e52b14, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=35.5 K 2024-11-13T03:53:14,721 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2bf67b67b148ae9716de0912e52b14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731469989541 2024-11-13T03:53:14,721 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 903c254eadab411ba1a772febac4f23b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731469990703 2024-11-13T03:53:14,721 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting e11337fd7f114e5dbf4f9bbc6705e8ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:14,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742367_1543 (size=31363) 2024-11-13T03:53:14,727 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:14,727 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/c414905b68844885aa4c467bc9d9545d is 50, key is test_row_0/C:col10/1731469992793/Put/seqid=0 2024-11-13T03:53:14,728 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/8057755f091f41bdbd40ce86f9fdf2b0 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0 2024-11-13T03:53:14,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742368_1544 (size=12409) 2024-11-13T03:53:14,733 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/A of d7159452becbf53ae4db2364b5f6792c into 8057755f091f41bdbd40ce86f9fdf2b0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:14,733 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:14,733 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/A, priority=13, startTime=1731469994683; duration=0sec 2024-11-13T03:53:14,733 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:14,733 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:A 2024-11-13T03:53:14,736 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/c414905b68844885aa4c467bc9d9545d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c414905b68844885aa4c467bc9d9545d 2024-11-13T03:53:14,739 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/C of d7159452becbf53ae4db2364b5f6792c into c414905b68844885aa4c467bc9d9545d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:14,739 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:14,739 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/C, priority=13, startTime=1731469994683; duration=0sec 2024-11-13T03:53:14,739 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:14,739 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:C 2024-11-13T03:53:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:14,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:14,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111315cdf9b1a0984bbda1e108571193e356_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469993477/Put/seqid=0 2024-11-13T03:53:14,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742369_1545 (size=12304) 2024-11-13T03:53:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470054894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470054894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470054895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470054896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:14,898 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T03:53:14,967 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:14,968 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T03:53:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:14,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:14,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:14,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470055006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470055006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470055007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:15,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470055216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470055216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470055217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:15,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:15,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111315cdf9b1a0984bbda1e108571193e356_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111315cdf9b1a0984bbda1e108571193e356_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:15,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/01f57fe4700a4f40a9e556687336913d, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:15,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/01f57fe4700a4f40a9e556687336913d is 175, key is test_row_0/A:col10/1731469993477/Put/seqid=0 2024-11-13T03:53:15,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742370_1546 (size=31105) 2024-11-13T03:53:15,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470055527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470055527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470055527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,562 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:53:15,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470055637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:15,639 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:15,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/01f57fe4700a4f40a9e556687336913d 2024-11-13T03:53:15,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/34aa5517552f4ee892538b71cb249e70 is 50, key is test_row_0/B:col10/1731469993477/Put/seqid=0 2024-11-13T03:53:15,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742371_1547 (size=12151) 2024-11-13T03:53:15,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:15,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470056037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470056037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470056037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:16,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:16,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/34aa5517552f4ee892538b71cb249e70 2024-11-13T03:53:16,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/00f94f1d531b4fc2aed91215ea855a2f is 50, key is test_row_0/C:col10/1731469993477/Put/seqid=0 2024-11-13T03:53:16,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742372_1548 (size=12151) 2024-11-13T03:53:16,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:16,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/00f94f1d531b4fc2aed91215ea855a2f 2024-11-13T03:53:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/01f57fe4700a4f40a9e556687336913d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d 2024-11-13T03:53:16,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d, entries=150, sequenceid=158, filesize=30.4 K 2024-11-13T03:53:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/34aa5517552f4ee892538b71cb249e70 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70 2024-11-13T03:53:16,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T03:53:16,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/00f94f1d531b4fc2aed91215ea855a2f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f 2024-11-13T03:53:16,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T03:53:16,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d7159452becbf53ae4db2364b5f6792c in 1653ms, sequenceid=158, compaction requested=false 2024-11-13T03:53:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:16,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:16,652 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:16,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111348d86882beeb43cc8497a3eefcd7a024_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469994882/Put/seqid=0 2024-11-13T03:53:16,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742373_1549 (size=12304) 2024-11-13T03:53:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:17,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:17,063 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111348d86882beeb43cc8497a3eefcd7a024_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111348d86882beeb43cc8497a3eefcd7a024_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:17,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/005ffd01708740708e4b17bccdbbcd49, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:17,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/005ffd01708740708e4b17bccdbbcd49 is 175, key is test_row_0/A:col10/1731469994882/Put/seqid=0 2024-11-13T03:53:17,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742374_1550 (size=31105) 2024-11-13T03:53:17,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470057071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470057074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470057075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470057187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470057187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470057187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470057396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470057396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470057396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,468 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/005ffd01708740708e4b17bccdbbcd49 2024-11-13T03:53:17,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/234051c88ad14fa1977160dd2748cba9 is 50, key is test_row_0/B:col10/1731469994882/Put/seqid=0 2024-11-13T03:53:17,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742375_1551 (size=12151) 2024-11-13T03:53:17,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470057656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,660 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470057707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470057707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470057707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:17,877 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/234051c88ad14fa1977160dd2748cba9 2024-11-13T03:53:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/a69b6bbd0a204a30ba6e18aa2bb3e350 is 50, key is test_row_0/C:col10/1731469994882/Put/seqid=0 2024-11-13T03:53:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742376_1552 (size=12151) 2024-11-13T03:53:17,886 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/a69b6bbd0a204a30ba6e18aa2bb3e350 2024-11-13T03:53:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/005ffd01708740708e4b17bccdbbcd49 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49 2024-11-13T03:53:17,891 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49, entries=150, sequenceid=171, filesize=30.4 K 2024-11-13T03:53:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/234051c88ad14fa1977160dd2748cba9 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9 2024-11-13T03:53:17,894 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T03:53:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/a69b6bbd0a204a30ba6e18aa2bb3e350 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350 2024-11-13T03:53:17,897 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T03:53:17,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d7159452becbf53ae4db2364b5f6792c in 1246ms, sequenceid=171, compaction requested=true 2024-11-13T03:53:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T03:53:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T03:53:17,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T03:53:17,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9290 sec 2024-11-13T03:53:17,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.9330 sec 2024-11-13T03:53:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:18,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:53:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:18,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:18,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:18,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc3ff88e2a7941a4b0dbde681c44bba8_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:18,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742377_1553 (size=14794) 2024-11-13T03:53:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470058231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470058232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470058234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470058347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470058347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470058347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470058557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470058557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470058557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:18,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc3ff88e2a7941a4b0dbde681c44bba8_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc3ff88e2a7941a4b0dbde681c44bba8_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:18,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/49961755fc784553829f6becfbe5201a, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:18,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/49961755fc784553829f6becfbe5201a is 175, key is test_row_0/A:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:18,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742378_1554 (size=39749) 2024-11-13T03:53:18,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/49961755fc784553829f6becfbe5201a 2024-11-13T03:53:18,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/a1ae7031ca9041e0912f77e9c3460b80 is 50, key is test_row_0/B:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:18,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742379_1555 (size=12151) 2024-11-13T03:53:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470058867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470058867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470058867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:18,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53628 deadline: 1731470058937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:18,940 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_2' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:19,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/a1ae7031ca9041e0912f77e9c3460b80 2024-11-13T03:53:19,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1a4ad1fd6f304d358ca1c9a07477bb14 is 50, key is test_row_0/C:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:19,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742380_1556 (size=12151) 2024-11-13T03:53:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T03:53:19,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:19,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T03:53:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:19,128 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:19,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:19,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:19,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T03:53:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470059377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470059377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470059377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:19,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T03:53:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:53:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:19,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1a4ad1fd6f304d358ca1c9a07477bb14 2024-11-13T03:53:19,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/49961755fc784553829f6becfbe5201a as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a 2024-11-13T03:53:19,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a, entries=200, sequenceid=191, filesize=38.8 K 2024-11-13T03:53:19,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/a1ae7031ca9041e0912f77e9c3460b80 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80 2024-11-13T03:53:19,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T03:53:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/1a4ad1fd6f304d358ca1c9a07477bb14 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14 2024-11-13T03:53:19,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T03:53:19,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d7159452becbf53ae4db2364b5f6792c in 1265ms, sequenceid=191, compaction requested=true 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:19,484 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:53:19,484 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:53:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:19,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:53:19,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/A is initiating minor compaction (all files) 2024-11-13T03:53:19,485 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/A in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,485 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=130.2 K 2024-11-13T03:53:19,485 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a] 2024-11-13T03:53:19,485 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:53:19,485 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8057755f091f41bdbd40ce86f9fdf2b0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/B is initiating minor compaction (all files) 2024-11-13T03:53:19,486 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/B in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,486 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/5d64901e163349ef90c9eb5f4b06bb7c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=47.7 K 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f57fe4700a4f40a9e556687336913d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731469993458 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d64901e163349ef90c9eb5f4b06bb7c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005ffd01708740708e4b17bccdbbcd49, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469994882 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 34aa5517552f4ee892538b71cb249e70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731469993458 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49961755fc784553829f6becfbe5201a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997068 2024-11-13T03:53:19,486 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 234051c88ad14fa1977160dd2748cba9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469994882 2024-11-13T03:53:19,487 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ae7031ca9041e0912f77e9c3460b80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997069 2024-11-13T03:53:19,491 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:19,493 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:19,494 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/d28f863ce2a849a1ba774cbfe787afa2 is 50, key is test_row_0/B:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:19,496 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111384a4d54c72104adf901f6e1b348b0fd5_d7159452becbf53ae4db2364b5f6792c store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:19,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742381_1557 (size=12595) 2024-11-13T03:53:19,499 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111384a4d54c72104adf901f6e1b348b0fd5_d7159452becbf53ae4db2364b5f6792c, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:19,499 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111384a4d54c72104adf901f6e1b348b0fd5_d7159452becbf53ae4db2364b5f6792c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742382_1558 (size=4469) 2024-11-13T03:53:19,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T03:53:19,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,586 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da4b36387aa34a858a8cdd2b69da2b4e_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731469998233/Put/seqid=0 2024-11-13T03:53:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742383_1559 (size=12304) 2024-11-13T03:53:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:19,900 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/d28f863ce2a849a1ba774cbfe787afa2 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/d28f863ce2a849a1ba774cbfe787afa2 2024-11-13T03:53:19,903 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/B of d7159452becbf53ae4db2364b5f6792c into d28f863ce2a849a1ba774cbfe787afa2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:19,903 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:19,903 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/B, priority=12, startTime=1731469999484; duration=0sec 2024-11-13T03:53:19,904 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:19,904 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:19,904 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:B 2024-11-13T03:53:19,904 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:53:19,904 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/318770b2da8449568474f7856d451408 is 175, key is test_row_0/A:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:19,905 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T03:53:19,905 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/C is initiating minor compaction (all files) 2024-11-13T03:53:19,905 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/C in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:19,906 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c414905b68844885aa4c467bc9d9545d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=47.7 K 2024-11-13T03:53:19,906 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c414905b68844885aa4c467bc9d9545d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731469992769 2024-11-13T03:53:19,906 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f94f1d531b4fc2aed91215ea855a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731469993458 2024-11-13T03:53:19,906 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting a69b6bbd0a204a30ba6e18aa2bb3e350, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731469994882 2024-11-13T03:53:19,907 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4ad1fd6f304d358ca1c9a07477bb14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997069 2024-11-13T03:53:19,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742384_1560 (size=31549) 2024-11-13T03:53:19,911 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/318770b2da8449568474f7856d451408 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408 2024-11-13T03:53:19,915 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:19,916 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/be4a63a938b84929a8b699ccd9be73c6 is 50, key is test_row_0/C:col10/1731469997069/Put/seqid=0 2024-11-13T03:53:19,920 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/A of d7159452becbf53ae4db2364b5f6792c into 318770b2da8449568474f7856d451408(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:19,920 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:19,920 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/A, priority=12, startTime=1731469999484; duration=0sec 2024-11-13T03:53:19,920 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:19,920 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:A 2024-11-13T03:53:19,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742385_1561 (size=12595) 2024-11-13T03:53:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:19,998 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da4b36387aa34a858a8cdd2b69da2b4e_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da4b36387aa34a858a8cdd2b69da2b4e_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/ed4673ba7aea4d278fa075f41c983bc8, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/ed4673ba7aea4d278fa075f41c983bc8 is 175, key is test_row_0/A:col10/1731469998233/Put/seqid=0 2024-11-13T03:53:20,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742386_1562 (size=31105) 2024-11-13T03:53:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:20,327 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/be4a63a938b84929a8b699ccd9be73c6 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/be4a63a938b84929a8b699ccd9be73c6 2024-11-13T03:53:20,331 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/C of d7159452becbf53ae4db2364b5f6792c into be4a63a938b84929a8b699ccd9be73c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:20,331 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:20,331 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/C, priority=12, startTime=1731469999484; duration=0sec 2024-11-13T03:53:20,331 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:20,331 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:C 2024-11-13T03:53:20,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:20,402 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/ed4673ba7aea4d278fa075f41c983bc8 2024-11-13T03:53:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/180a4c787c79434f8156ba57c42e3887 is 50, key is test_row_0/B:col10/1731469998233/Put/seqid=0 2024-11-13T03:53:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470060410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470060411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470060412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742387_1563 (size=12151) 2024-11-13T03:53:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470060517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470060527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470060527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470060727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470060737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470060737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:20,824 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/180a4c787c79434f8156ba57c42e3887 2024-11-13T03:53:20,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/c7311f96fed54392a0bba0e2aa5b4ea3 is 50, key is test_row_0/C:col10/1731469998233/Put/seqid=0 2024-11-13T03:53:20,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742388_1564 (size=12151) 2024-11-13T03:53:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470061036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470061046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470061047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/c7311f96fed54392a0bba0e2aa5b4ea3 2024-11-13T03:53:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/ed4673ba7aea4d278fa075f41c983bc8 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8 2024-11-13T03:53:21,240 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8, entries=150, sequenceid=203, filesize=30.4 K 2024-11-13T03:53:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/180a4c787c79434f8156ba57c42e3887 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887 2024-11-13T03:53:21,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T03:53:21,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/c7311f96fed54392a0bba0e2aa5b4ea3 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3 2024-11-13T03:53:21,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T03:53:21,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d7159452becbf53ae4db2364b5f6792c in 1661ms, sequenceid=203, compaction requested=false 2024-11-13T03:53:21,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:21,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:21,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T03:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T03:53:21,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T03:53:21,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1190 sec 2024-11-13T03:53:21,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1230 sec 2024-11-13T03:53:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T03:53:21,276 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:21,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T03:53:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:21,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:21,279 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:21,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:21,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T03:53:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:21,431 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f7a468ffa1b40a49110fce03a249f18_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731470000411/Put/seqid=0 2024-11-13T03:53:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742389_1565 (size=12304) 2024-11-13T03:53:21,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:21,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470061565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470061566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470061567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470061677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470061677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470061677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53652 deadline: 1731470061686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,689 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ac1a51731247,42123,1731469817167 for 'test_row_1' in d7159452becbf53ae4db2364b5f6792c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T03:53:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:21,851 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f7a468ffa1b40a49110fce03a249f18_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f7a468ffa1b40a49110fce03a249f18_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/94103d295f034c2fbfe6335dfb6ffe4f, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/94103d295f034c2fbfe6335dfb6ffe4f is 175, key is test_row_0/A:col10/1731470000411/Put/seqid=0 2024-11-13T03:53:21,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742390_1566 (size=31105) 2024-11-13T03:53:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470061887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470061887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470061887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470062196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:22,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470062196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470062197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:22,256 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/94103d295f034c2fbfe6335dfb6ffe4f 2024-11-13T03:53:22,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/20e6094b09f24f62979b75f0690a4657 is 50, key is test_row_0/B:col10/1731470000411/Put/seqid=0 2024-11-13T03:53:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742391_1567 (size=12151) 2024-11-13T03:53:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:22,665 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/20e6094b09f24f62979b75f0690a4657 2024-11-13T03:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9522333357b34cc7b2b816b4f12b093c is 50, key is test_row_0/C:col10/1731470000411/Put/seqid=0 2024-11-13T03:53:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742392_1568 (size=12151) 2024-11-13T03:53:22,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470062706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470062707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470062707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,086 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9522333357b34cc7b2b816b4f12b093c 2024-11-13T03:53:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/94103d295f034c2fbfe6335dfb6ffe4f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f 2024-11-13T03:53:23,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f, entries=150, sequenceid=226, filesize=30.4 K 2024-11-13T03:53:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/20e6094b09f24f62979b75f0690a4657 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657 2024-11-13T03:53:23,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T03:53:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9522333357b34cc7b2b816b4f12b093c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c 2024-11-13T03:53:23,149 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T03:53:23,150 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d7159452becbf53ae4db2364b5f6792c in 1718ms, sequenceid=226, compaction requested=true 2024-11-13T03:53:23,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:23,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:23,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T03:53:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T03:53:23,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T03:53:23,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-13T03:53:23,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8830 sec 2024-11-13T03:53:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T03:53:23,427 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:23,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T03:53:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T03:53:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:23,429 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T03:53:23,429 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T03:53:23,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T03:53:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:23,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T03:53:23,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:23,582 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366e45e86c6e8490fb656509425f8710b_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731470001565/Put/seqid=0 2024-11-13T03:53:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742393_1569 (size=12304) 2024-11-13T03:53:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:23,595 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366e45e86c6e8490fb656509425f8710b_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366e45e86c6e8490fb656509425f8710b_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/9713ad6435294e809110fba072afe5bf, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:23,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/9713ad6435294e809110fba072afe5bf is 175, key is test_row_0/A:col10/1731470001565/Put/seqid=0 2024-11-13T03:53:23,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742394_1570 (size=31105) 2024-11-13T03:53:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. as already flushing 2024-11-13T03:53:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470063744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470063745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470063750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470063857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470063857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:23,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470063867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,001 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/9713ad6435294e809110fba072afe5bf 2024-11-13T03:53:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/54a88e7195544940a69b2c17cb1ff511 is 50, key is test_row_0/B:col10/1731470001565/Put/seqid=0 2024-11-13T03:53:24,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742395_1571 (size=12151) 2024-11-13T03:53:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470064066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470064066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470064077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53590 deadline: 1731470064377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1731470064377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T03:53:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53604 deadline: 1731470064387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 2024-11-13T03:53:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 , the old value is region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T03:53:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., hostname=ac1a51731247,42123,1731469817167, seqNum=5 because the exception is null or not the one we care about 2024-11-13T03:53:24,411 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/54a88e7195544940a69b2c17cb1ff511 2024-11-13T03:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/3bab0d6ab64c4d2bad5da5c169d83173 is 50, key is test_row_0/C:col10/1731470001565/Put/seqid=0 2024-11-13T03:53:24,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742396_1572 (size=12151) 2024-11-13T03:53:24,499 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-13T03:53:24,499 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-13T03:53:24,499 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,499 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,499 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,499 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,499 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,499 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,501 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-13T03:53:24,501 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,501 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,501 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,503 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-13T03:53:24,503 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,503 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,503 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,503 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-13T03:53:24,503 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,503 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,503 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:24,820 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/3bab0d6ab64c4d2bad5da5c169d83173 2024-11-13T03:53:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/9713ad6435294e809110fba072afe5bf as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf 2024-11-13T03:53:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf, entries=150, sequenceid=238, filesize=30.4 K 2024-11-13T03:53:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/54a88e7195544940a69b2c17cb1ff511 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511 2024-11-13T03:53:24,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T03:53:24,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/3bab0d6ab64c4d2bad5da5c169d83173 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173 2024-11-13T03:53:24,844 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T03:53:24,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d7159452becbf53ae4db2364b5f6792c in 1264ms, sequenceid=238, compaction requested=true 2024-11-13T03:53:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ac1a51731247:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T03:53:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T03:53:24,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T03:53:24,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-13T03:53:24,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4210 sec 2024-11-13T03:53:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42123 {}] regionserver.HRegion(8855): Flush requested on d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:24,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T03:53:24,891 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-13T03:53:24,891 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-13T03:53:24,891 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,891 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:24,892 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,892 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:24,892 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,892 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T03:53:24,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,898 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-13T03:53:24,898 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:24,898 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,898 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:24,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:24,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e96b10b08924a35b4a0dc1d14a94fd6_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_0/A:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742397_1573 (size=12404) 2024-11-13T03:53:25,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:25,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e96b10b08924a35b4a0dc1d14a94fd6_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e96b10b08924a35b4a0dc1d14a94fd6_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:25,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c21f2564d4504c61ad54da68ffc211f1, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:25,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c21f2564d4504c61ad54da68ffc211f1 is 175, key is test_row_0/A:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:25,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742398_1574 (size=31205) 2024-11-13T03:53:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T03:53:25,578 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:25,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c21f2564d4504c61ad54da68ffc211f1 2024-11-13T03:53:25,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2e38130009e347e3bd76a29f9c3ace39 is 50, key is test_row_0/B:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:25,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742399_1575 (size=12251) 2024-11-13T03:53:26,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2e38130009e347e3bd76a29f9c3ace39 2024-11-13T03:53:26,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fa5cb8eea1f2431fa526c79e686dbd8b is 50, key is test_row_0/C:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:26,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742400_1576 (size=12251) 2024-11-13T03:53:26,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fa5cb8eea1f2431fa526c79e686dbd8b 2024-11-13T03:53:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/c21f2564d4504c61ad54da68ffc211f1 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1 2024-11-13T03:53:26,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1, entries=150, sequenceid=260, filesize=30.5 K 2024-11-13T03:53:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/2e38130009e347e3bd76a29f9c3ace39 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39 2024-11-13T03:53:26,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T03:53:26,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/fa5cb8eea1f2431fa526c79e686dbd8b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b 2024-11-13T03:53:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T03:53:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=6.71 KB/6870 for d7159452becbf53ae4db2364b5f6792c in 1693ms, sequenceid=260, compaction requested=true 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:26,585 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:26,585 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7159452becbf53ae4db2364b5f6792c:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T03:53:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:26,586 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156069 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T03:53:26,586 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/A is initiating minor compaction (all files) 2024-11-13T03:53:26,586 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61299 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T03:53:26,587 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/A in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:26,587 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/B is initiating minor compaction (all files) 2024-11-13T03:53:26,587 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/B in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:26,587 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=152.4 K 2024-11-13T03:53:26,587 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:26,587 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/d28f863ce2a849a1ba774cbfe787afa2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=59.9 K 2024-11-13T03:53:26,587 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. files: [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1] 2024-11-13T03:53:26,587 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28f863ce2a849a1ba774cbfe787afa2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997069 2024-11-13T03:53:26,587 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 318770b2da8449568474f7856d451408, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997069 2024-11-13T03:53:26,587 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180a4c787c79434f8156ba57c42e3887, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731469998224 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4673ba7aea4d278fa075f41c983bc8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731469998224 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e6094b09f24f62979b75f0690a4657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731470000405 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 94103d295f034c2fbfe6335dfb6ffe4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731470000405 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54a88e7195544940a69b2c17cb1ff511, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731470001561 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting 9713ad6435294e809110fba072afe5bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731470001561 2024-11-13T03:53:26,588 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e38130009e347e3bd76a29f9c3ace39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731470003743 2024-11-13T03:53:26,589 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] compactions.Compactor(225): Compacting c21f2564d4504c61ad54da68ffc211f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731470003743 2024-11-13T03:53:26,598 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:26,599 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113a99e55e49edb4a78b7339ffdc246c6f9_d7159452becbf53ae4db2364b5f6792c store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:26,600 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#B#compaction#486 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:26,600 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/23daf3b4bdc64ebc9a1cd2a6695ff19f is 50, key is test_row_0/B:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:26,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742401_1577 (size=12865) 2024-11-13T03:53:26,604 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113a99e55e49edb4a78b7339ffdc246c6f9_d7159452becbf53ae4db2364b5f6792c, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:26,604 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a99e55e49edb4a78b7339ffdc246c6f9_d7159452becbf53ae4db2364b5f6792c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:26,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742402_1578 (size=4469) 2024-11-13T03:53:27,010 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:27,012 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e992b9c433244317adaa67486b36b729 is 175, key is test_row_0/A:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:27,016 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/23daf3b4bdc64ebc9a1cd2a6695ff19f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/23daf3b4bdc64ebc9a1cd2a6695ff19f 2024-11-13T03:53:27,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742403_1579 (size=31819) 2024-11-13T03:53:27,021 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/B of d7159452becbf53ae4db2364b5f6792c into 23daf3b4bdc64ebc9a1cd2a6695ff19f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:27,021 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:27,021 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/B, priority=11, startTime=1731470006585; duration=0sec 2024-11-13T03:53:27,021 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T03:53:27,021 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:B 2024-11-13T03:53:27,021 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T03:53:27,023 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61299 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T03:53:27,023 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1541): d7159452becbf53ae4db2364b5f6792c/C is initiating minor compaction (all files) 2024-11-13T03:53:27,023 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7159452becbf53ae4db2364b5f6792c/C in TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:27,023 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/be4a63a938b84929a8b699ccd9be73c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b] into tmpdir=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp, totalSize=59.9 K 2024-11-13T03:53:27,024 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4a63a938b84929a8b699ccd9be73c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731469997069 2024-11-13T03:53:27,024 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7311f96fed54392a0bba0e2aa5b4ea3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731469998224 2024-11-13T03:53:27,024 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9522333357b34cc7b2b816b4f12b093c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731470000405 2024-11-13T03:53:27,024 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bab0d6ab64c4d2bad5da5c169d83173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731470001561 2024-11-13T03:53:27,025 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5cb8eea1f2431fa526c79e686dbd8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731470003743 2024-11-13T03:53:27,032 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7159452becbf53ae4db2364b5f6792c#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:53:27,033 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/3480b164f46241df83f35ce35fea9cde is 50, key is test_row_0/C:col10/1731470003749/Put/seqid=0 2024-11-13T03:53:27,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742404_1580 (size=12865) 2024-11-13T03:53:27,431 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/e992b9c433244317adaa67486b36b729 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e992b9c433244317adaa67486b36b729 2024-11-13T03:53:27,437 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/A of d7159452becbf53ae4db2364b5f6792c into e992b9c433244317adaa67486b36b729(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:27,437 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:27,437 INFO [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/A, priority=11, startTime=1731470006585; duration=0sec 2024-11-13T03:53:27,437 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:27,437 DEBUG [RS:0;ac1a51731247:42123-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:A 2024-11-13T03:53:27,441 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/3480b164f46241df83f35ce35fea9cde as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3480b164f46241df83f35ce35fea9cde 2024-11-13T03:53:27,446 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d7159452becbf53ae4db2364b5f6792c/C of d7159452becbf53ae4db2364b5f6792c into 3480b164f46241df83f35ce35fea9cde(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:53:27,446 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7159452becbf53ae4db2364b5f6792c: 2024-11-13T03:53:27,446 INFO [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., storeName=d7159452becbf53ae4db2364b5f6792c/C, priority=11, startTime=1731470006585; duration=0sec 2024-11-13T03:53:27,446 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T03:53:27,446 DEBUG [RS:0;ac1a51731247:42123-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7159452becbf53ae4db2364b5f6792c:C 2024-11-13T03:53:29,001 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-13T03:53:29,001 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:29,002 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:29,002 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:29,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:31,782 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-13T03:53:31,782 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T03:53:31,782 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:31,782 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:31,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3661 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3484 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3375 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3664 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3487 2024-11-13T03:53:31,783 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T03:53:31,784 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:53:31,784 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:53:31,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:31,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:31,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:31,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T03:53:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:31,790 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731470011790"}]},"ts":"1731470011790"} 2024-11-13T03:53:31,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T03:53:31,792 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T03:53:31,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T03:53:31,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, UNASSIGN}] 2024-11-13T03:53:31,795 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, UNASSIGN 2024-11-13T03:53:31,796 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=CLOSING, regionLocation=ac1a51731247,42123,1731469817167 2024-11-13T03:53:31,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, UNASSIGN because future has completed 2024-11-13T03:53:31,798 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:53:31,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167}] 2024-11-13T03:53:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:31,952 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:31,952 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:53:31,953 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing d7159452becbf53ae4db2364b5f6792c, disabling compactions & flushes 2024-11-13T03:53:31,953 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:31,953 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:31,953 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. after waiting 0 ms 2024-11-13T03:53:31,953 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:31,953 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing d7159452becbf53ae4db2364b5f6792c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=A 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=B 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7159452becbf53ae4db2364b5f6792c, store=C 2024-11-13T03:53:31,954 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T03:53:31,964 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111346ace281609940428d463698bbb53cca_d7159452becbf53ae4db2364b5f6792c is 50, key is test_row_1/A:col10/1731470011779/Put/seqid=0 2024-11-13T03:53:31,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742405_1581 (size=9914) 2024-11-13T03:53:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:32,369 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:53:32,377 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111346ace281609940428d463698bbb53cca_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346ace281609940428d463698bbb53cca_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:32,378 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/74324ae321ad4569a8e8afb4ba9c1e7d, store: [table=TestAcidGuarantees family=A region=d7159452becbf53ae4db2364b5f6792c] 2024-11-13T03:53:32,379 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/74324ae321ad4569a8e8afb4ba9c1e7d is 175, key is test_row_1/A:col10/1731470011779/Put/seqid=0 2024-11-13T03:53:32,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742406_1582 (size=22561) 2024-11-13T03:53:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:32,784 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/74324ae321ad4569a8e8afb4ba9c1e7d 2024-11-13T03:53:32,797 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/82b68e5a4a7d42e7b0ca92f8492a688e is 50, key is test_row_1/B:col10/1731470011779/Put/seqid=0 2024-11-13T03:53:32,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742407_1583 (size=9857) 2024-11-13T03:53:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:33,202 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/82b68e5a4a7d42e7b0ca92f8492a688e 2024-11-13T03:53:33,215 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9a732f59014e4f119f00e61730ae2067 is 50, key is test_row_1/C:col10/1731470011779/Put/seqid=0 2024-11-13T03:53:33,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742408_1584 (size=9857) 2024-11-13T03:53:33,621 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9a732f59014e4f119f00e61730ae2067 2024-11-13T03:53:33,677 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/A/74324ae321ad4569a8e8afb4ba9c1e7d as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/74324ae321ad4569a8e8afb4ba9c1e7d 2024-11-13T03:53:33,685 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/74324ae321ad4569a8e8afb4ba9c1e7d, entries=100, sequenceid=269, filesize=22.0 K 2024-11-13T03:53:33,686 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/B/82b68e5a4a7d42e7b0ca92f8492a688e as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/82b68e5a4a7d42e7b0ca92f8492a688e 2024-11-13T03:53:33,689 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/82b68e5a4a7d42e7b0ca92f8492a688e, entries=100, sequenceid=269, filesize=9.6 K 2024-11-13T03:53:33,690 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/.tmp/C/9a732f59014e4f119f00e61730ae2067 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9a732f59014e4f119f00e61730ae2067 2024-11-13T03:53:33,694 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9a732f59014e4f119f00e61730ae2067, entries=100, sequenceid=269, filesize=9.6 K 2024-11-13T03:53:33,695 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d7159452becbf53ae4db2364b5f6792c in 1742ms, sequenceid=269, compaction requested=false 2024-11-13T03:53:33,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1] to archive 2024-11-13T03:53:33,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:33,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/0f28e0e421124d78b57f6cc30a7caba0 2024-11-13T03:53:33,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c7ad8246a28f4d199bc872e2f61af44b 2024-11-13T03:53:33,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/f433b9e2ad4740be8a360e893ce61933 2024-11-13T03:53:33,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/de557ba49db24d04b7c177afb2efb64b 2024-11-13T03:53:33,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/07a62a369ba2406b81472a53a6e50789 2024-11-13T03:53:33,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/782a59718b2643a4b4c1d368783db181 2024-11-13T03:53:33,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/6a1e30edd3b94b658874fa68dae8ddf0 2024-11-13T03:53:33,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/079292030f224f95983129e2146d373c 2024-11-13T03:53:33,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e11cccef148646a396c585bacd2025cf 2024-11-13T03:53:33,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/8057755f091f41bdbd40ce86f9fdf2b0 2024-11-13T03:53:33,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/01f57fe4700a4f40a9e556687336913d 2024-11-13T03:53:33,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/005ffd01708740708e4b17bccdbbcd49 2024-11-13T03:53:33,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/49961755fc784553829f6becfbe5201a 2024-11-13T03:53:33,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/318770b2da8449568474f7856d451408 2024-11-13T03:53:33,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/ed4673ba7aea4d278fa075f41c983bc8 2024-11-13T03:53:33,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/94103d295f034c2fbfe6335dfb6ffe4f 2024-11-13T03:53:33,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/9713ad6435294e809110fba072afe5bf 2024-11-13T03:53:33,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/c21f2564d4504c61ad54da68ffc211f1 2024-11-13T03:53:33,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/005f7e16df1e4b709b6e90a801dcd9de, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/cf5ad3b54ef0431dbf82653bfdc2c771, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/5d64901e163349ef90c9eb5f4b06bb7c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/d28f863ce2a849a1ba774cbfe787afa2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39] to archive 2024-11-13T03:53:33,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:33,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2ac3725c70c94eb1bf201c48240e306b 2024-11-13T03:53:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/aa42c75584cc48e79cacc4400005e512 2024-11-13T03:53:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/005f7e16df1e4b709b6e90a801dcd9de to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/005f7e16df1e4b709b6e90a801dcd9de 2024-11-13T03:53:33,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/f5bae0bafbc549afb9492c0a5ff48ebc 2024-11-13T03:53:33,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/67154ea5a32248449d79405cc717ae00 2024-11-13T03:53:33,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/cf5ad3b54ef0431dbf82653bfdc2c771 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/cf5ad3b54ef0431dbf82653bfdc2c771 2024-11-13T03:53:33,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/c94cd0ce90314526a35e839b0cf2fa4e 2024-11-13T03:53:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/306953ad7ba04ff7985ce7a9721a6245 2024-11-13T03:53:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/5d64901e163349ef90c9eb5f4b06bb7c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/5d64901e163349ef90c9eb5f4b06bb7c 2024-11-13T03:53:33,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/55eae16d33f84bc0aaad9a785d3637d4 2024-11-13T03:53:33,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/34aa5517552f4ee892538b71cb249e70 2024-11-13T03:53:33,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/234051c88ad14fa1977160dd2748cba9 2024-11-13T03:53:33,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/d28f863ce2a849a1ba774cbfe787afa2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/d28f863ce2a849a1ba774cbfe787afa2 2024-11-13T03:53:33,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/a1ae7031ca9041e0912f77e9c3460b80 2024-11-13T03:53:33,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/180a4c787c79434f8156ba57c42e3887 2024-11-13T03:53:33,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/20e6094b09f24f62979b75f0690a4657 2024-11-13T03:53:33,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/54a88e7195544940a69b2c17cb1ff511 2024-11-13T03:53:33,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/2e38130009e347e3bd76a29f9c3ace39 2024-11-13T03:53:33,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e20a46d3f2cc4da5a2bb284c9a589dec, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/7e2bf67b67b148ae9716de0912e52b14, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c414905b68844885aa4c467bc9d9545d, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/be4a63a938b84929a8b699ccd9be73c6, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b] to archive 2024-11-13T03:53:33,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:53:33,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/ed581c5faa724e22adce59e10e867b9c 2024-11-13T03:53:33,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fdeeafb67f6640f6abf8541adf17cc6a 2024-11-13T03:53:33,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e20a46d3f2cc4da5a2bb284c9a589dec to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e20a46d3f2cc4da5a2bb284c9a589dec 2024-11-13T03:53:33,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/07e155bed6cc487b902a83610906b1af 2024-11-13T03:53:33,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1ebe6c5665244fd986e26acd94c64cf2 2024-11-13T03:53:33,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/7e2bf67b67b148ae9716de0912e52b14 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/7e2bf67b67b148ae9716de0912e52b14 2024-11-13T03:53:33,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/b4eecc3c3cf24a6b8cb9021806cabe8d 2024-11-13T03:53:33,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/903c254eadab411ba1a772febac4f23b 2024-11-13T03:53:33,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c414905b68844885aa4c467bc9d9545d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c414905b68844885aa4c467bc9d9545d 2024-11-13T03:53:33,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/e11337fd7f114e5dbf4f9bbc6705e8ad 2024-11-13T03:53:33,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/00f94f1d531b4fc2aed91215ea855a2f 2024-11-13T03:53:33,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/a69b6bbd0a204a30ba6e18aa2bb3e350 2024-11-13T03:53:33,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/be4a63a938b84929a8b699ccd9be73c6 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/be4a63a938b84929a8b699ccd9be73c6 2024-11-13T03:53:33,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/1a4ad1fd6f304d358ca1c9a07477bb14 2024-11-13T03:53:33,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/c7311f96fed54392a0bba0e2aa5b4ea3 2024-11-13T03:53:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9522333357b34cc7b2b816b4f12b093c 2024-11-13T03:53:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3bab0d6ab64c4d2bad5da5c169d83173 2024-11-13T03:53:33,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/fa5cb8eea1f2431fa526c79e686dbd8b 2024-11-13T03:53:33,760 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits/272.seqid, newMaxSeqId=272, maxSeqId=4 2024-11-13T03:53:33,760 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. 2024-11-13T03:53:33,760 DEBUG [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for d7159452becbf53ae4db2364b5f6792c: Waiting for close lock at 1731470011953Running coprocessor pre-close hooks at 1731470011953Disabling compacts and flushes for region at 1731470011953Disabling writes for close at 1731470011953Obtaining lock to block concurrent updates at 1731470011953Preparing flush snapshotting stores in d7159452becbf53ae4db2364b5f6792c at 1731470011953Finished memstore snapshotting TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731470011954 (+1 ms)Flushing stores of TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c. at 1731470011956 (+2 ms)Flushing d7159452becbf53ae4db2364b5f6792c/A: creating writer at 1731470011956Flushing mob file d7159452becbf53ae4db2364b5f6792c/A: appending metadata at 1731470011964 (+8 ms)Flushing mob file d7159452becbf53ae4db2364b5f6792c/A: closing flushed file at 1731470011964Flushing d7159452becbf53ae4db2364b5f6792c/A: appending metadata at 1731470012378 (+414 ms)Flushing d7159452becbf53ae4db2364b5f6792c/A: closing flushed file at 1731470012378Flushing d7159452becbf53ae4db2364b5f6792c/B: creating writer at 1731470012794 (+416 ms)Flushing d7159452becbf53ae4db2364b5f6792c/B: appending metadata at 1731470012796 (+2 ms)Flushing d7159452becbf53ae4db2364b5f6792c/B: closing flushed file at 1731470012796Flushing d7159452becbf53ae4db2364b5f6792c/C: creating writer at 1731470013211 (+415 ms)Flushing d7159452becbf53ae4db2364b5f6792c/C: appending metadata at 1731470013214 (+3 ms)Flushing d7159452becbf53ae4db2364b5f6792c/C: closing flushed file at 1731470013214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a54b588: reopening flushed file at 1731470013675 (+461 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24bbce2c: reopening flushed file at 1731470013685 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@614626c0: reopening flushed file at 1731470013690 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d7159452becbf53ae4db2364b5f6792c in 1742ms, sequenceid=269, compaction requested=false at 1731470013695 (+5 ms)Writing region close event to WAL at 1731470013757 (+62 ms)Running coprocessor post-close hooks at 1731470013760 (+3 ms)Closed at 1731470013760 2024-11-13T03:53:33,762 INFO [RS_CLOSE_REGION-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=d7159452becbf53ae4db2364b5f6792c, regionState=CLOSED 2024-11-13T03:53:33,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 because future has completed 2024-11-13T03:53:33,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T03:53:33,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure d7159452becbf53ae4db2364b5f6792c, server=ac1a51731247,42123,1731469817167 in 1.9660 sec 2024-11-13T03:53:33,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-13T03:53:33,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7159452becbf53ae4db2364b5f6792c, UNASSIGN in 1.9710 sec 2024-11-13T03:53:33,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T03:53:33,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9740 sec 2024-11-13T03:53:33,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731470013769"}]},"ts":"1731470013769"} 2024-11-13T03:53:33,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T03:53:33,770 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T03:53:33,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9850 sec 2024-11-13T03:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T03:53:33,948 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:33,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T03:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:33,950 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T03:53:33,951 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:33,954 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,956 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C, FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits] 2024-11-13T03:53:33,960 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/74324ae321ad4569a8e8afb4ba9c1e7d to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/74324ae321ad4569a8e8afb4ba9c1e7d 2024-11-13T03:53:33,961 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e992b9c433244317adaa67486b36b729 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/A/e992b9c433244317adaa67486b36b729 2024-11-13T03:53:33,964 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/23daf3b4bdc64ebc9a1cd2a6695ff19f to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/23daf3b4bdc64ebc9a1cd2a6695ff19f 2024-11-13T03:53:33,965 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/82b68e5a4a7d42e7b0ca92f8492a688e to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/B/82b68e5a4a7d42e7b0ca92f8492a688e 2024-11-13T03:53:33,968 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3480b164f46241df83f35ce35fea9cde to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/3480b164f46241df83f35ce35fea9cde 2024-11-13T03:53:33,970 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9a732f59014e4f119f00e61730ae2067 to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/C/9a732f59014e4f119f00e61730ae2067 2024-11-13T03:53:33,973 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits/272.seqid to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c/recovered.edits/272.seqid 2024-11-13T03:53:33,974 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/default/TestAcidGuarantees/d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,974 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T03:53:33,974 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:53:33,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T03:53:33,979 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130563e9e455034aabb0855f0df672fcee_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130563e9e455034aabb0855f0df672fcee_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111315cdf9b1a0984bbda1e108571193e356_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111315cdf9b1a0984bbda1e108571193e356_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,981 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f7a468ffa1b40a49110fce03a249f18_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f7a468ffa1b40a49110fce03a249f18_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,982 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346ace281609940428d463698bbb53cca_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111346ace281609940428d463698bbb53cca_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111348d86882beeb43cc8497a3eefcd7a024_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111348d86882beeb43cc8497a3eefcd7a024_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113657eced0b72c43aa9a71492456eb98aa_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113657eced0b72c43aa9a71492456eb98aa_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,985 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366e45e86c6e8490fb656509425f8710b_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366e45e86c6e8490fb656509425f8710b_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,986 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e96b10b08924a35b4a0dc1d14a94fd6_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e96b10b08924a35b4a0dc1d14a94fd6_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113835fd5ebcdb046d8a397a7e0c72022c3_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113835fd5ebcdb046d8a397a7e0c72022c3_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aa22fd3efa14d60a6eac6769df3c35a_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aa22fd3efa14d60a6eac6769df3c35a_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139ce2f41488a04853bbbd9869aef46188_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139ce2f41488a04853bbbd9869aef46188_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d364f5ae5e5491fab47687dfd3c7a74_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d364f5ae5e5491fab47687dfd3c7a74_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa29000c849b401689f1672bd49c2521_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa29000c849b401689f1672bd49c2521_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da4b36387aa34a858a8cdd2b69da2b4e_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113da4b36387aa34a858a8cdd2b69da2b4e_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc3ff88e2a7941a4b0dbde681c44bba8_d7159452becbf53ae4db2364b5f6792c to hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc3ff88e2a7941a4b0dbde681c44bba8_d7159452becbf53ae4db2364b5f6792c 2024-11-13T03:53:33,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T03:53:33,995 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:33,998 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T03:53:34,000 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T03:53:34,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:34,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T03:53:34,002 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731470014001"}]},"ts":"9223372036854775807"} 2024-11-13T03:53:34,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T03:53:34,004 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d7159452becbf53ae4db2364b5f6792c, NAME => 'TestAcidGuarantees,,1731469981346.d7159452becbf53ae4db2364b5f6792c.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T03:53:34,004 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T03:53:34,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731470014004"}]},"ts":"9223372036854775807"} 2024-11-13T03:53:34,006 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T03:53:34,007 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T03:53:34,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-13T03:53:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T03:53:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T03:53:34,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T03:53:34,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 247) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 310), ProcessCount=11 (was 11), AvailableMemoryMB=8093 (was 8124) 2024-11-13T03:53:34,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T03:53:34,073 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:53:34,073 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:53:34,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:34,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:34,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:53:34,074 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T03:53:34,074 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=974344401, stopped=false 2024-11-13T03:53:34,075 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ac1a51731247,33933,1731469816394 2024-11-13T03:53:34,124 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:53:34,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:53:34,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:53:34,124 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:53:34,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:53:34,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:53:34,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:53:34,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:34,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:53:34,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:53:34,127 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ac1a51731247,42123,1731469817167' ***** 2024-11-13T03:53:34,127 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T03:53:34,128 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:53:34,128 INFO [RS:0;ac1a51731247:42123 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T03:53:34,128 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:53:34,128 INFO [RS:0;ac1a51731247:42123 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(959): stopping server ac1a51731247,42123,1731469817167 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ac1a51731247:42123. 2024-11-13T03:53:34,129 DEBUG [RS:0;ac1a51731247:42123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:53:34,129 DEBUG [RS:0;ac1a51731247:42123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:53:34,129 INFO [RS:0;ac1a51731247:42123 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:53:34,130 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T03:53:34,130 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:53:34,131 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T03:53:34,131 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:34,131 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:53:34,131 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:53:34,131 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:53:34,131 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:53:34,131 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:53:34,131 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T03:53:34,153 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/info/1ffa1cffc19f4384afceda1d3377bfe5 is 95, key is TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e./info:/1731469854962/DeleteFamily/seqid=0 2024-11-13T03:53:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742409_1585 (size=6514) 2024-11-13T03:53:34,198 INFO [regionserver/ac1a51731247:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:53:34,331 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:34,388 INFO [regionserver/ac1a51731247:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T03:53:34,388 INFO [regionserver/ac1a51731247:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T03:53:34,532 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:34,558 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/info/1ffa1cffc19f4384afceda1d3377bfe5 2024-11-13T03:53:34,586 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/ns/df5d60b0f0eb48e8b4bc8da129c84998 is 93, key is TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e./ns:/1731469854921/DeleteFamily/seqid=0 2024-11-13T03:53:34,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742410_1586 (size=5906) 2024-11-13T03:53:34,732 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:34,932 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:34,991 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/ns/df5d60b0f0eb48e8b4bc8da129c84998 2024-11-13T03:53:35,018 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/rep_barrier/03161ed028d64c9bac4a487a2cb4ca1b is 102, key is TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e./rep_barrier:/1731469854921/DeleteFamily/seqid=0 2024-11-13T03:53:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742411_1587 (size=6025) 2024-11-13T03:53:35,133 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:35,333 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:35,423 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/rep_barrier/03161ed028d64c9bac4a487a2cb4ca1b 2024-11-13T03:53:35,449 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/table/e2c0a834c6bd40dbb653c99152293901 is 96, key is TestAcidGuarantees,,1731469821005.40c7504f29b68767a43cbb0fe4bac67e./table:/1731469854921/DeleteFamily/seqid=0 2024-11-13T03:53:35,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742412_1588 (size=5936) 2024-11-13T03:53:35,533 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:35,733 DEBUG [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:53:35,854 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/table/e2c0a834c6bd40dbb653c99152293901 2024-11-13T03:53:35,863 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/info/1ffa1cffc19f4384afceda1d3377bfe5 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/info/1ffa1cffc19f4384afceda1d3377bfe5 2024-11-13T03:53:35,867 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/info/1ffa1cffc19f4384afceda1d3377bfe5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T03:53:35,868 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/ns/df5d60b0f0eb48e8b4bc8da129c84998 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/ns/df5d60b0f0eb48e8b4bc8da129c84998 2024-11-13T03:53:35,871 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/ns/df5d60b0f0eb48e8b4bc8da129c84998, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T03:53:35,872 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/rep_barrier/03161ed028d64c9bac4a487a2cb4ca1b as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/rep_barrier/03161ed028d64c9bac4a487a2cb4ca1b 2024-11-13T03:53:35,875 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/rep_barrier/03161ed028d64c9bac4a487a2cb4ca1b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T03:53:35,876 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/.tmp/table/e2c0a834c6bd40dbb653c99152293901 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/table/e2c0a834c6bd40dbb653c99152293901 2024-11-13T03:53:35,879 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/table/e2c0a834c6bd40dbb653c99152293901, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T03:53:35,879 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1748ms, sequenceid=90, compaction requested=false 2024-11-13T03:53:35,883 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T03:53:35,883 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:53:35,884 INFO [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:53:35,884 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731470014130Running coprocessor pre-close hooks at 1731470014131 (+1 ms)Disabling compacts and flushes for region at 1731470014131Disabling writes for close at 1731470014131Obtaining lock to block concurrent updates at 1731470014131Preparing flush snapshotting stores in 1588230740 at 1731470014131Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731470014132 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731470014133 (+1 ms)Flushing 1588230740/info: creating writer at 1731470014133Flushing 1588230740/info: appending metadata at 1731470014152 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731470014152Flushing 1588230740/ns: creating writer at 1731470014568 (+416 ms)Flushing 1588230740/ns: appending metadata at 1731470014586 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731470014586Flushing 1588230740/rep_barrier: creating writer at 1731470015000 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731470015018 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731470015018Flushing 1588230740/table: creating writer at 1731470015433 (+415 ms)Flushing 1588230740/table: appending metadata at 1731470015449 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731470015449Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9000f8: reopening flushed file at 1731470015863 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@329c47ec: reopening flushed file at 1731470015867 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78fad99d: reopening flushed file at 1731470015871 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31d92267: reopening flushed file at 1731470015875 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1748ms, sequenceid=90, compaction requested=false at 1731470015879 (+4 ms)Writing region close event to WAL at 1731470015881 (+2 ms)Running coprocessor post-close hooks at 1731470015883 (+2 ms)Closed at 1731470015884 (+1 ms) 2024-11-13T03:53:35,884 DEBUG [RS_CLOSE_META-regionserver/ac1a51731247:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T03:53:35,934 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(976): stopping server ac1a51731247,42123,1731469817167; all regions closed. 2024-11-13T03:53:35,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741834_1010 (size=25344) 2024-11-13T03:53:35,943 DEBUG [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/oldWALs 2024-11-13T03:53:35,943 INFO [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ac1a51731247%2C42123%2C1731469817167.meta:.meta(num 1731469820518) 2024-11-13T03:53:35,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741832_1008 (size=15525736) 2024-11-13T03:53:35,949 DEBUG [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/oldWALs 2024-11-13T03:53:35,949 INFO [RS:0;ac1a51731247:42123 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ac1a51731247%2C42123%2C1731469817167:(num 1731469819553) 2024-11-13T03:53:35,950 DEBUG [RS:0;ac1a51731247:42123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:53:35,950 INFO [RS:0;ac1a51731247:42123 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:53:35,950 INFO [RS:0;ac1a51731247:42123 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:53:35,950 INFO [RS:0;ac1a51731247:42123 {}] hbase.ChoreService(370): Chore service for: regionserver/ac1a51731247:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T03:53:35,951 INFO [RS:0;ac1a51731247:42123 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:53:35,951 INFO [regionserver/ac1a51731247:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:53:35,952 INFO [RS:0;ac1a51731247:42123 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42123 2024-11-13T03:53:36,006 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ac1a51731247,42123,1731469817167 2024-11-13T03:53:36,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:53:36,007 INFO [RS:0;ac1a51731247:42123 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:53:36,008 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ac1a51731247,42123,1731469817167] 2024-11-13T03:53:36,023 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ac1a51731247,42123,1731469817167 already deleted, retry=false 2024-11-13T03:53:36,023 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ac1a51731247,42123,1731469817167 expired; onlineServers=0 2024-11-13T03:53:36,024 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ac1a51731247,33933,1731469816394' ***** 2024-11-13T03:53:36,024 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T03:53:36,024 INFO [M:0;ac1a51731247:33933 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:53:36,024 INFO [M:0;ac1a51731247:33933 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:53:36,024 DEBUG [M:0;ac1a51731247:33933 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T03:53:36,025 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T03:53:36,025 DEBUG [M:0;ac1a51731247:33933 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T03:53:36,025 DEBUG [master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.large.0-1731469819185 {}] cleaner.HFileCleaner(306): Exit Thread[master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.large.0-1731469819185,5,FailOnTimeoutGroup] 2024-11-13T03:53:36,025 DEBUG [master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.small.0-1731469819188 {}] cleaner.HFileCleaner(306): Exit Thread[master/ac1a51731247:0:becomeActiveMaster-HFileCleaner.small.0-1731469819188,5,FailOnTimeoutGroup] 2024-11-13T03:53:36,026 INFO [M:0;ac1a51731247:33933 {}] hbase.ChoreService(370): Chore service for: master/ac1a51731247:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T03:53:36,026 INFO [M:0;ac1a51731247:33933 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:53:36,026 DEBUG [M:0;ac1a51731247:33933 {}] master.HMaster(1795): Stopping service threads 2024-11-13T03:53:36,026 INFO [M:0;ac1a51731247:33933 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T03:53:36,026 INFO [M:0;ac1a51731247:33933 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:53:36,027 ERROR [M:0;ac1a51731247:33933 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44905 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44905,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T03:53:36,027 INFO [M:0;ac1a51731247:33933 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T03:53:36,027 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T03:53:36,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T03:53:36,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:53:36,031 DEBUG [M:0;ac1a51731247:33933 {}] zookeeper.ZKUtil(347): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T03:53:36,031 WARN [M:0;ac1a51731247:33933 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T03:53:36,031 INFO [M:0;ac1a51731247:33933 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/.lastflushedseqids 2024-11-13T03:53:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742413_1589 (size=72) 2024-11-13T03:53:36,116 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:53:36,116 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42123-0x101339a42000001, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:53:36,117 INFO [RS:0;ac1a51731247:42123 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:53:36,117 INFO [RS:0;ac1a51731247:42123 {}] regionserver.HRegionServer(1031): Exiting; stopping=ac1a51731247,42123,1731469817167; zookeeper connection closed. 2024-11-13T03:53:36,118 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63248f6f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63248f6f 2024-11-13T03:53:36,119 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T03:53:36,442 INFO [M:0;ac1a51731247:33933 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T03:53:36,442 INFO [M:0;ac1a51731247:33933 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T03:53:36,443 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:53:36,443 INFO [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:53:36,443 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:53:36,443 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:53:36,443 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:53:36,443 INFO [M:0;ac1a51731247:33933 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=746.22 KB heapSize=912.04 KB 2024-11-13T03:53:36,464 DEBUG [M:0;ac1a51731247:33933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f626061961b4b2396e3cd1575e3b605 is 82, key is hbase:meta,,1/info:regioninfo/1731469820607/Put/seqid=0 2024-11-13T03:53:36,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742414_1590 (size=5672) 2024-11-13T03:53:36,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:53:36,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T03:53:36,760 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T03:53:36,868 INFO [M:0;ac1a51731247:33933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f626061961b4b2396e3cd1575e3b605 2024-11-13T03:53:36,898 DEBUG [M:0;ac1a51731247:33933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd7160651bb84680ab18a55459eb0a0f is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1731469920656/Put/seqid=0 2024-11-13T03:53:36,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742415_1591 (size=44997) 2024-11-13T03:53:37,303 INFO [M:0;ac1a51731247:33933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=745.66 KB at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd7160651bb84680ab18a55459eb0a0f 2024-11-13T03:53:37,312 INFO [M:0;ac1a51731247:33933 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dd7160651bb84680ab18a55459eb0a0f 2024-11-13T03:53:37,330 DEBUG [M:0;ac1a51731247:33933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f84a87df6744e2bd6b12232f3e712c is 69, key is ac1a51731247,42123,1731469817167/rs:state/1731469819311/Put/seqid=0 2024-11-13T03:53:37,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073742416_1592 (size=5156) 2024-11-13T03:53:37,734 INFO [M:0;ac1a51731247:33933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f84a87df6744e2bd6b12232f3e712c 2024-11-13T03:53:37,746 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f626061961b4b2396e3cd1575e3b605 as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5f626061961b4b2396e3cd1575e3b605 2024-11-13T03:53:37,749 INFO [M:0;ac1a51731247:33933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5f626061961b4b2396e3cd1575e3b605, entries=8, sequenceid=2037, filesize=5.5 K 2024-11-13T03:53:37,750 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dd7160651bb84680ab18a55459eb0a0f as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dd7160651bb84680ab18a55459eb0a0f 2024-11-13T03:53:37,753 INFO [M:0;ac1a51731247:33933 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dd7160651bb84680ab18a55459eb0a0f 2024-11-13T03:53:37,753 INFO [M:0;ac1a51731247:33933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dd7160651bb84680ab18a55459eb0a0f, entries=168, sequenceid=2037, filesize=43.9 K 2024-11-13T03:53:37,753 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f84a87df6744e2bd6b12232f3e712c as hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4f84a87df6744e2bd6b12232f3e712c 2024-11-13T03:53:37,756 INFO [M:0;ac1a51731247:33933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44905/user/jenkins/test-data/9818e133-d793-117e-711c-719b1194958e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4f84a87df6744e2bd6b12232f3e712c, entries=1, sequenceid=2037, filesize=5.0 K 2024-11-13T03:53:37,757 INFO [M:0;ac1a51731247:33933 {}] regionserver.HRegion(3140): Finished flush of dataSize ~746.22 KB/764125, heapSize ~911.74 KB/933624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=2037, compaction requested=false 2024-11-13T03:53:37,758 INFO [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:53:37,759 DEBUG [M:0;ac1a51731247:33933 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731470016443Disabling compacts and flushes for region at 1731470016443Disabling writes for close at 1731470016443Obtaining lock to block concurrent updates at 1731470016443Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731470016443Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=764125, getHeapSize=933864, getOffHeapSize=0, getCellsCount=2230 at 1731470016444 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731470016445 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731470016445Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731470016464 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731470016464Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731470016879 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731470016897 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731470016897Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731470017312 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731470017329 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731470017329Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a1b191a: reopening flushed file at 1731470017745 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4be3646f: reopening flushed file at 1731470017749 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4687248c: reopening flushed file at 1731470017753 (+4 ms)Finished flush of dataSize ~746.22 KB/764125, heapSize ~911.74 KB/933624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=2037, compaction requested=false at 1731470017757 (+4 ms)Writing region close event to WAL at 1731470017758 (+1 ms)Closed at 1731470017758 2024-11-13T03:53:37,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38833 is added to blk_1073741830_1006 (size=897721) 2024-11-13T03:53:37,762 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:53:37,762 INFO [M:0;ac1a51731247:33933 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T03:53:37,762 INFO [M:0;ac1a51731247:33933 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33933 2024-11-13T03:53:37,762 INFO [M:0;ac1a51731247:33933 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:53:37,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:53:37,915 INFO [M:0;ac1a51731247:33933 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:53:37,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33933-0x101339a42000000, quorum=127.0.0.1:62776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:53:37,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:53:37,930 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:53:37,930 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:53:37,930 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:53:37,931 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.log.dir/,STOPPED} 2024-11-13T03:53:37,933 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:53:37,933 WARN [BP-784585066-172.17.0.2-1731469812880 heartbeating to localhost/127.0.0.1:44905 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:53:37,933 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:53:37,933 WARN [BP-784585066-172.17.0.2-1731469812880 heartbeating to localhost/127.0.0.1:44905 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-784585066-172.17.0.2-1731469812880 (Datanode Uuid 98147724-73a0-4f09-96cb-5b744e935ac9) service to localhost/127.0.0.1:44905 2024-11-13T03:53:37,936 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data1/current/BP-784585066-172.17.0.2-1731469812880 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:53:37,937 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/cluster_ba05a183-6b85-a16e-8815-d092dca942e8/data/data2/current/BP-784585066-172.17.0.2-1731469812880 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:53:37,937 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:53:37,945 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:53:37,946 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:53:37,946 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:53:37,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:53:37,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a6c1620-29f5-c221-92bd-21e351404a6d/hadoop.log.dir/,STOPPED} 2024-11-13T03:53:37,958 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T03:53:38,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down